{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'\\x00\\x00\\x00\\x02'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import struct\n",
    "struct.pack('>i',2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 获取数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1、数据来源是字节数组，解析字节数组（拆包）获得数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2051, 60000, 28, 28)\n",
      "47040000\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import struct\n",
    "with open('./MNIST_data/train-images-idx3-ubyte', 'rb') as f:\n",
    "    buffer1 = f.read(4*4)\n",
    "    header = struct.unpack('>iiii', buffer1)\n",
    "    print(header)\n",
    "    piexls_length = header[1] * header[2] * header[3]\n",
    "    print(piexls_length)\n",
    "    buffer = f.read(piexls_length)\n",
    "    data = struct.unpack('>{}B'.format(piexls_length),buffer)\n",
    "    imgs = np.reshape(data, (header[1], header[2], header[3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOZ0lEQVR4nO3dbYxc5XnG8euKbezamMQbB9chLjjgFAg0Jl0ZEBZQoVCCKgGqArGiyKG0ThOchNaVoLQqtKKVWyVElFIkU1xMxUsgAeEPNAm1ECRqcFlcY2wIb8Y0NmaNWYENIX5Z3/2w42iBnWeXmTMv3vv/k1Yzc+45c24NXD5nznNmHkeEAIx/H+p0AwDag7ADSRB2IAnCDiRB2IEkJrZzY4d5ckzRtHZuEkjlV3pbe2OPR6o1FXbb50m6QdIESf8WEctLz5+iaTrV5zSzSQAFa2NN3VrDh/G2J0i6SdLnJZ0oaZHtExt9PQCt1cxn9gWSXoiIzRGxV9Ldki6opi0AVWsm7EdJ+sWwx1try97F9hLbfbb79mlPE5sD0IyWn42PiBUR0RsRvZM0udWbA1BHM2HfJmnOsMefqC0D0IWaCfvjkubZnmv7MElflLS6mrYAVK3hobeI2G97qaQfaWjobWVEbKqsMwCVamqcPSIelPRgRb0AaCEulwWSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJpmZxRffzxPJ/4gkfm9nS7T/7F8fUrQ1OPVBc9+hjdxTrU7/uYv3V6w+rW1vX+73iujsH3y7WT713WbF+3J8/Vqx3QlNht71F0m5Jg5L2R0RvFU0BqF4Ve/bfi4idFbwOgBbiMzuQRLNhD0k/tv2E7SUjPcH2Ett9tvv2aU+TmwPQqGYP4xdGxDbbR0p6yPbPI+LR4U+IiBWSVkjSEe6JJrcHoEFN7dkjYlvtdoek+yUtqKIpANVrOOy2p9mefvC+pHMlbayqMQDVauYwfpak+20ffJ07I+KHlXQ1zkw4YV6xHpMnFeuvnPWRYv2d0+qPCfd8uDxe/JPPlMebO+k/fzm9WP/HfzmvWF978p11ay/te6e47vL+zxXrH//JofeJtOGwR8RmSZ+psBcALcTQG5AEYQeSIOxAEoQdSIKwA0nwFdcKDJ792WL9+ttuKtY/Nan+VzHHs30xWKz/zY1fKdYnvl0e/jr93qV1a9O37S+uO3lneWhuat/aYr0bsWcHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQYZ6/A5GdfKdaf+NWcYv1Tk/qrbKdSy7afVqxvfqv8U9S3Hfv9urU3D5THyWf9838X66106H2BdXTs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCUe0b0TxCPfEqT6nbdvrFgOXnl6s7zqv/HPPEzYcXqw/+fUbP3BPB12383eK9cfPKo+jD77xZrEep9f/AeIt3yyuqrmLniw/Ae+zNtZoVwyMOJc1e3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9i4wYeZHi/XB1weK9ZfurD9WvunMlcV1F/zDN4r1I2/q3HfK8cE1Nc5ue6XtHbY3DlvWY/sh28/XbmdU2TCA6o3lMP42Se+d9f4qSWsiYp6kNbXHALrYqGGPiEclvfc48gJJq2r3V0m6sNq2AFSt0d+gmxUR22v3X5U0q94TbS+RtESSpmhqg5sD0Kymz8bH0Bm+umf5ImJFRPRGRO8kTW52cwAa1GjY+23PlqTa7Y7qWgLQCo2GfbWkxbX7iyU9UE07AFpl1M/stu+SdLakmba3SrpG0nJJ99i+TNLLki5uZZPj3eDO15taf9+uxud3//SXni7WX7t5QvkFDpTnWEf3GDXsEbGoTomrY4BDCJfLAkkQdiAJwg4kQdiBJAg7kARTNo8DJ1z5XN3apSeXB03+/eg1xfpZX7i8WJ/+vceKdXQP9uxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7ONAadrk1792QnHd/1v9TrF+1XW3F+t/efFFxXr874fr1ub8/c+K66qNP3OeAXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCKZuTG/ij04v1O675drE+d+KUhrf96duXFuvzbtlerO/fvKXhbY9XTU3ZDGB8IOxAEoQdSIKwA0kQdiAJwg4kQdiBJBhnR1GcMb9YP2L51mL9rk/+qOFtH//wHxfrv/239b/HL0mDz29ueNuHqqbG2W2vtL3D9sZhy661vc32+trf+VU2DKB6YzmMv03SeSMs/25EzK/9PVhtWwCqNmrYI+JRSQNt6AVACzVzgm6p7Q21w/wZ9Z5ke4ntPtt9+7Snic0BaEajYb9Z0rGS5kvaLuk79Z4YESsiojcieidpcoObA9CshsIeEf0RMRgRByTdImlBtW0BqFpDYbc9e9jDiyRtrPdcAN1h1HF223dJOlvSTEn9kq6pPZ4vKSRtkfTViCh/+ViMs49HE2YdWay/cslxdWtrr7yhuO6HRtkXfemlc4v1Nxe+XqyPR6Vx9lEniYiIRSMsvrXprgC0FZfLAkkQdiAJwg4kQdiBJAg7kARfcUXH3LO1PGXzVB9WrP8y9hbrf/CNK+q/9v1ri+seqvgpaQCEHciCsANJEHYgCcIOJEHYgSQIO5DEqN96Q24HFs4v1l/8QnnK5pPmb6lbG20cfTQ3DpxSrE99oK+p1x9v2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs49z7j2pWH/um+Wx7lvOWFWsnzml/J3yZuyJfcX6YwNzyy9wYNRfN0+FPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4+yFg4tyji/UXL/143dq1l9xdXPcPD9/ZUE9VuLq/t1h/5IbTivUZq8q/O493G3XPbnuO7YdtP217k+1v1Zb32H7I9vO12xmtbxdAo8ZyGL9f0rKIOFHSaZIut32ipKskrYmIeZLW1B4D6FKjhj0itkfEutr93ZKekXSUpAskHbyWcpWkC1vUI4AKfKDP7LaPkXSKpLWSZkXEwYuPX5U0q846SyQtkaQpmtpwowCaM+az8bYPl/QDSVdExK7htRiaHXLEGSIjYkVE9EZE7yRNbqpZAI0bU9htT9JQ0O+IiPtqi/ttz67VZ0va0ZoWAVRh1MN425Z0q6RnIuL6YaXVkhZLWl67faAlHY4DE4/5rWL9zd+dXaxf8nc/LNb/9CP3FeuttGx7eXjsZ/9af3it57b/Ka474wBDa1Uay2f2MyR9WdJTttfXll2toZDfY/sySS9LurglHQKoxKhhj4ifShpxcndJ51TbDoBW4XJZIAnCDiRB2IEkCDuQBGEHkuArrmM0cfZv1q0NrJxWXPdrcx8p1hdN72+opyos3bawWF938/xifeb3NxbrPbsZK+8W7NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IIk04+x7f7/8s8V7/2ygWL/6uAfr1s79jbcb6qkq/YPv1K2duXpZcd3j//rnxXrPG+Vx8gPFKroJe3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSCLNOPuWC8v/rj138r0t2/ZNbxxbrN/wyLnFugfr/bjvkOOve6lubV7/2uK6g8UqxhP27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQhCOi/AR7jqTbJc2SFJJWRMQNtq+V9CeSXqs99eqIqP+lb0lHuCdONRO/Aq2yNtZoVwyMeGHGWC6q2S9pWUSssz1d0hO2H6rVvhsR366qUQCtM5b52bdL2l67v9v2M5KOanVjAKr1gT6z2z5G0imSDl6DudT2Btsrbc+os84S2322+/ZpT3PdAmjYmMNu+3BJP5B0RUTsknSzpGMlzdfQnv87I60XESsiojcieidpcvMdA2jImMJue5KGgn5HRNwnSRHRHxGDEXFA0i2SFrSuTQDNGjXsti3pVknPRMT1w5bPHva0iySVp/ME0FFjORt/hqQvS3rK9vrasqslLbI9X0PDcVskfbUF/QGoyFjOxv9U0kjjdsUxdQDdhSvogCQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSYz6U9KVbsx+TdLLwxbNlLSzbQ18MN3aW7f2JdFbo6rs7eiI+NhIhbaG/X0bt/siordjDRR0a2/d2pdEb41qV28cxgNJEHYgiU6HfUWHt1/Srb11a18SvTWqLb119DM7gPbp9J4dQJsQdiCJjoTd9nm2n7X9gu2rOtFDPba32H7K9nrbfR3uZaXtHbY3DlvWY/sh28/XbkecY69DvV1re1vtvVtv+/wO9TbH9sO2n7a9yfa3ass7+t4V+mrL+9b2z+y2J0h6TtLnJG2V9LikRRHxdFsbqcP2Fkm9EdHxCzBsnynpLUm3R8RJtWX/JGkgIpbX/qGcERFXdklv10p6q9PTeNdmK5o9fJpxSRdK+oo6+N4V+rpYbXjfOrFnXyDphYjYHBF7Jd0t6YIO9NH1IuJRSQPvWXyBpFW1+6s09D9L29XprStExPaIWFe7v1vSwWnGO/reFfpqi06E/ShJvxj2eKu6a773kPRj20/YXtLpZkYwKyK21+6/KmlWJ5sZwajTeLfTe6YZ75r3rpHpz5vFCbr3WxgRn5X0eUmX1w5Xu1IMfQbrprHTMU3j3S4jTDP+a5187xqd/rxZnQj7Nklzhj3+RG1ZV4iIbbXbHZLuV/dNRd1/cAbd2u2ODvfza900jfdI04yrC967Tk5/3omwPy5pnu25tg+T9EVJqzvQx/vYnlY7cSLb0ySdq+6binq1pMW1+4slPdDBXt6lW6bxrjfNuDr83nV8+vOIaPufpPM1dEb+RUl/1Yke6vT1SUlP1v42dbo3SXdp6LBun4bObVwm6aOS1kh6XtJ/Serpot7+Q9JTkjZoKFizO9TbQg0dom+QtL72d36n37tCX21537hcFkiCE3RAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kMT/A65XcTMQuIbWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "# imshow显示图片\n",
    "plt.imshow(imgs[0])\n",
    "plt.show()   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2、数据来源是mat,直接读取数据即可"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.datasets import fetch_mldata  \n",
    "\n",
    "# mnist = fetch_mldata('MNIST original', data_home = './')\n",
    "# mnist\n",
    "\n",
    "# sklearn版本问题，使用fetch_openml读取数据\n",
    "from sklearn.datasets import fetch_openml\n",
    "mnist = fetch_openml('mnist_784')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'data': array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]]),\n",
       " 'target': array(['5', '0', '4', ..., '4', '5', '6'], dtype=object),\n",
       " 'frame': None,\n",
       " 'categories': {},\n",
       " 'feature_names': ['pixel1',\n",
       "  'pixel2',\n",
       "  'pixel3',\n",
       "  'pixel4',\n",
       "  'pixel5',\n",
       "  'pixel6',\n",
       "  'pixel7',\n",
       "  'pixel8',\n",
       "  'pixel9',\n",
       "  'pixel10',\n",
       "  'pixel11',\n",
       "  'pixel12',\n",
       "  'pixel13',\n",
       "  'pixel14',\n",
       "  'pixel15',\n",
       "  'pixel16',\n",
       "  'pixel17',\n",
       "  'pixel18',\n",
       "  'pixel19',\n",
       "  'pixel20',\n",
       "  'pixel21',\n",
       "  'pixel22',\n",
       "  'pixel23',\n",
       "  'pixel24',\n",
       "  'pixel25',\n",
       "  'pixel26',\n",
       "  'pixel27',\n",
       "  'pixel28',\n",
       "  'pixel29',\n",
       "  'pixel30',\n",
       "  'pixel31',\n",
       "  'pixel32',\n",
       "  'pixel33',\n",
       "  'pixel34',\n",
       "  'pixel35',\n",
       "  'pixel36',\n",
       "  'pixel37',\n",
       "  'pixel38',\n",
       "  'pixel39',\n",
       "  'pixel40',\n",
       "  'pixel41',\n",
       "  'pixel42',\n",
       "  'pixel43',\n",
       "  'pixel44',\n",
       "  'pixel45',\n",
       "  'pixel46',\n",
       "  'pixel47',\n",
       "  'pixel48',\n",
       "  'pixel49',\n",
       "  'pixel50',\n",
       "  'pixel51',\n",
       "  'pixel52',\n",
       "  'pixel53',\n",
       "  'pixel54',\n",
       "  'pixel55',\n",
       "  'pixel56',\n",
       "  'pixel57',\n",
       "  'pixel58',\n",
       "  'pixel59',\n",
       "  'pixel60',\n",
       "  'pixel61',\n",
       "  'pixel62',\n",
       "  'pixel63',\n",
       "  'pixel64',\n",
       "  'pixel65',\n",
       "  'pixel66',\n",
       "  'pixel67',\n",
       "  'pixel68',\n",
       "  'pixel69',\n",
       "  'pixel70',\n",
       "  'pixel71',\n",
       "  'pixel72',\n",
       "  'pixel73',\n",
       "  'pixel74',\n",
       "  'pixel75',\n",
       "  'pixel76',\n",
       "  'pixel77',\n",
       "  'pixel78',\n",
       "  'pixel79',\n",
       "  'pixel80',\n",
       "  'pixel81',\n",
       "  'pixel82',\n",
       "  'pixel83',\n",
       "  'pixel84',\n",
       "  'pixel85',\n",
       "  'pixel86',\n",
       "  'pixel87',\n",
       "  'pixel88',\n",
       "  'pixel89',\n",
       "  'pixel90',\n",
       "  'pixel91',\n",
       "  'pixel92',\n",
       "  'pixel93',\n",
       "  'pixel94',\n",
       "  'pixel95',\n",
       "  'pixel96',\n",
       "  'pixel97',\n",
       "  'pixel98',\n",
       "  'pixel99',\n",
       "  'pixel100',\n",
       "  'pixel101',\n",
       "  'pixel102',\n",
       "  'pixel103',\n",
       "  'pixel104',\n",
       "  'pixel105',\n",
       "  'pixel106',\n",
       "  'pixel107',\n",
       "  'pixel108',\n",
       "  'pixel109',\n",
       "  'pixel110',\n",
       "  'pixel111',\n",
       "  'pixel112',\n",
       "  'pixel113',\n",
       "  'pixel114',\n",
       "  'pixel115',\n",
       "  'pixel116',\n",
       "  'pixel117',\n",
       "  'pixel118',\n",
       "  'pixel119',\n",
       "  'pixel120',\n",
       "  'pixel121',\n",
       "  'pixel122',\n",
       "  'pixel123',\n",
       "  'pixel124',\n",
       "  'pixel125',\n",
       "  'pixel126',\n",
       "  'pixel127',\n",
       "  'pixel128',\n",
       "  'pixel129',\n",
       "  'pixel130',\n",
       "  'pixel131',\n",
       "  'pixel132',\n",
       "  'pixel133',\n",
       "  'pixel134',\n",
       "  'pixel135',\n",
       "  'pixel136',\n",
       "  'pixel137',\n",
       "  'pixel138',\n",
       "  'pixel139',\n",
       "  'pixel140',\n",
       "  'pixel141',\n",
       "  'pixel142',\n",
       "  'pixel143',\n",
       "  'pixel144',\n",
       "  'pixel145',\n",
       "  'pixel146',\n",
       "  'pixel147',\n",
       "  'pixel148',\n",
       "  'pixel149',\n",
       "  'pixel150',\n",
       "  'pixel151',\n",
       "  'pixel152',\n",
       "  'pixel153',\n",
       "  'pixel154',\n",
       "  'pixel155',\n",
       "  'pixel156',\n",
       "  'pixel157',\n",
       "  'pixel158',\n",
       "  'pixel159',\n",
       "  'pixel160',\n",
       "  'pixel161',\n",
       "  'pixel162',\n",
       "  'pixel163',\n",
       "  'pixel164',\n",
       "  'pixel165',\n",
       "  'pixel166',\n",
       "  'pixel167',\n",
       "  'pixel168',\n",
       "  'pixel169',\n",
       "  'pixel170',\n",
       "  'pixel171',\n",
       "  'pixel172',\n",
       "  'pixel173',\n",
       "  'pixel174',\n",
       "  'pixel175',\n",
       "  'pixel176',\n",
       "  'pixel177',\n",
       "  'pixel178',\n",
       "  'pixel179',\n",
       "  'pixel180',\n",
       "  'pixel181',\n",
       "  'pixel182',\n",
       "  'pixel183',\n",
       "  'pixel184',\n",
       "  'pixel185',\n",
       "  'pixel186',\n",
       "  'pixel187',\n",
       "  'pixel188',\n",
       "  'pixel189',\n",
       "  'pixel190',\n",
       "  'pixel191',\n",
       "  'pixel192',\n",
       "  'pixel193',\n",
       "  'pixel194',\n",
       "  'pixel195',\n",
       "  'pixel196',\n",
       "  'pixel197',\n",
       "  'pixel198',\n",
       "  'pixel199',\n",
       "  'pixel200',\n",
       "  'pixel201',\n",
       "  'pixel202',\n",
       "  'pixel203',\n",
       "  'pixel204',\n",
       "  'pixel205',\n",
       "  'pixel206',\n",
       "  'pixel207',\n",
       "  'pixel208',\n",
       "  'pixel209',\n",
       "  'pixel210',\n",
       "  'pixel211',\n",
       "  'pixel212',\n",
       "  'pixel213',\n",
       "  'pixel214',\n",
       "  'pixel215',\n",
       "  'pixel216',\n",
       "  'pixel217',\n",
       "  'pixel218',\n",
       "  'pixel219',\n",
       "  'pixel220',\n",
       "  'pixel221',\n",
       "  'pixel222',\n",
       "  'pixel223',\n",
       "  'pixel224',\n",
       "  'pixel225',\n",
       "  'pixel226',\n",
       "  'pixel227',\n",
       "  'pixel228',\n",
       "  'pixel229',\n",
       "  'pixel230',\n",
       "  'pixel231',\n",
       "  'pixel232',\n",
       "  'pixel233',\n",
       "  'pixel234',\n",
       "  'pixel235',\n",
       "  'pixel236',\n",
       "  'pixel237',\n",
       "  'pixel238',\n",
       "  'pixel239',\n",
       "  'pixel240',\n",
       "  'pixel241',\n",
       "  'pixel242',\n",
       "  'pixel243',\n",
       "  'pixel244',\n",
       "  'pixel245',\n",
       "  'pixel246',\n",
       "  'pixel247',\n",
       "  'pixel248',\n",
       "  'pixel249',\n",
       "  'pixel250',\n",
       "  'pixel251',\n",
       "  'pixel252',\n",
       "  'pixel253',\n",
       "  'pixel254',\n",
       "  'pixel255',\n",
       "  'pixel256',\n",
       "  'pixel257',\n",
       "  'pixel258',\n",
       "  'pixel259',\n",
       "  'pixel260',\n",
       "  'pixel261',\n",
       "  'pixel262',\n",
       "  'pixel263',\n",
       "  'pixel264',\n",
       "  'pixel265',\n",
       "  'pixel266',\n",
       "  'pixel267',\n",
       "  'pixel268',\n",
       "  'pixel269',\n",
       "  'pixel270',\n",
       "  'pixel271',\n",
       "  'pixel272',\n",
       "  'pixel273',\n",
       "  'pixel274',\n",
       "  'pixel275',\n",
       "  'pixel276',\n",
       "  'pixel277',\n",
       "  'pixel278',\n",
       "  'pixel279',\n",
       "  'pixel280',\n",
       "  'pixel281',\n",
       "  'pixel282',\n",
       "  'pixel283',\n",
       "  'pixel284',\n",
       "  'pixel285',\n",
       "  'pixel286',\n",
       "  'pixel287',\n",
       "  'pixel288',\n",
       "  'pixel289',\n",
       "  'pixel290',\n",
       "  'pixel291',\n",
       "  'pixel292',\n",
       "  'pixel293',\n",
       "  'pixel294',\n",
       "  'pixel295',\n",
       "  'pixel296',\n",
       "  'pixel297',\n",
       "  'pixel298',\n",
       "  'pixel299',\n",
       "  'pixel300',\n",
       "  'pixel301',\n",
       "  'pixel302',\n",
       "  'pixel303',\n",
       "  'pixel304',\n",
       "  'pixel305',\n",
       "  'pixel306',\n",
       "  'pixel307',\n",
       "  'pixel308',\n",
       "  'pixel309',\n",
       "  'pixel310',\n",
       "  'pixel311',\n",
       "  'pixel312',\n",
       "  'pixel313',\n",
       "  'pixel314',\n",
       "  'pixel315',\n",
       "  'pixel316',\n",
       "  'pixel317',\n",
       "  'pixel318',\n",
       "  'pixel319',\n",
       "  'pixel320',\n",
       "  'pixel321',\n",
       "  'pixel322',\n",
       "  'pixel323',\n",
       "  'pixel324',\n",
       "  'pixel325',\n",
       "  'pixel326',\n",
       "  'pixel327',\n",
       "  'pixel328',\n",
       "  'pixel329',\n",
       "  'pixel330',\n",
       "  'pixel331',\n",
       "  'pixel332',\n",
       "  'pixel333',\n",
       "  'pixel334',\n",
       "  'pixel335',\n",
       "  'pixel336',\n",
       "  'pixel337',\n",
       "  'pixel338',\n",
       "  'pixel339',\n",
       "  'pixel340',\n",
       "  'pixel341',\n",
       "  'pixel342',\n",
       "  'pixel343',\n",
       "  'pixel344',\n",
       "  'pixel345',\n",
       "  'pixel346',\n",
       "  'pixel347',\n",
       "  'pixel348',\n",
       "  'pixel349',\n",
       "  'pixel350',\n",
       "  'pixel351',\n",
       "  'pixel352',\n",
       "  'pixel353',\n",
       "  'pixel354',\n",
       "  'pixel355',\n",
       "  'pixel356',\n",
       "  'pixel357',\n",
       "  'pixel358',\n",
       "  'pixel359',\n",
       "  'pixel360',\n",
       "  'pixel361',\n",
       "  'pixel362',\n",
       "  'pixel363',\n",
       "  'pixel364',\n",
       "  'pixel365',\n",
       "  'pixel366',\n",
       "  'pixel367',\n",
       "  'pixel368',\n",
       "  'pixel369',\n",
       "  'pixel370',\n",
       "  'pixel371',\n",
       "  'pixel372',\n",
       "  'pixel373',\n",
       "  'pixel374',\n",
       "  'pixel375',\n",
       "  'pixel376',\n",
       "  'pixel377',\n",
       "  'pixel378',\n",
       "  'pixel379',\n",
       "  'pixel380',\n",
       "  'pixel381',\n",
       "  'pixel382',\n",
       "  'pixel383',\n",
       "  'pixel384',\n",
       "  'pixel385',\n",
       "  'pixel386',\n",
       "  'pixel387',\n",
       "  'pixel388',\n",
       "  'pixel389',\n",
       "  'pixel390',\n",
       "  'pixel391',\n",
       "  'pixel392',\n",
       "  'pixel393',\n",
       "  'pixel394',\n",
       "  'pixel395',\n",
       "  'pixel396',\n",
       "  'pixel397',\n",
       "  'pixel398',\n",
       "  'pixel399',\n",
       "  'pixel400',\n",
       "  'pixel401',\n",
       "  'pixel402',\n",
       "  'pixel403',\n",
       "  'pixel404',\n",
       "  'pixel405',\n",
       "  'pixel406',\n",
       "  'pixel407',\n",
       "  'pixel408',\n",
       "  'pixel409',\n",
       "  'pixel410',\n",
       "  'pixel411',\n",
       "  'pixel412',\n",
       "  'pixel413',\n",
       "  'pixel414',\n",
       "  'pixel415',\n",
       "  'pixel416',\n",
       "  'pixel417',\n",
       "  'pixel418',\n",
       "  'pixel419',\n",
       "  'pixel420',\n",
       "  'pixel421',\n",
       "  'pixel422',\n",
       "  'pixel423',\n",
       "  'pixel424',\n",
       "  'pixel425',\n",
       "  'pixel426',\n",
       "  'pixel427',\n",
       "  'pixel428',\n",
       "  'pixel429',\n",
       "  'pixel430',\n",
       "  'pixel431',\n",
       "  'pixel432',\n",
       "  'pixel433',\n",
       "  'pixel434',\n",
       "  'pixel435',\n",
       "  'pixel436',\n",
       "  'pixel437',\n",
       "  'pixel438',\n",
       "  'pixel439',\n",
       "  'pixel440',\n",
       "  'pixel441',\n",
       "  'pixel442',\n",
       "  'pixel443',\n",
       "  'pixel444',\n",
       "  'pixel445',\n",
       "  'pixel446',\n",
       "  'pixel447',\n",
       "  'pixel448',\n",
       "  'pixel449',\n",
       "  'pixel450',\n",
       "  'pixel451',\n",
       "  'pixel452',\n",
       "  'pixel453',\n",
       "  'pixel454',\n",
       "  'pixel455',\n",
       "  'pixel456',\n",
       "  'pixel457',\n",
       "  'pixel458',\n",
       "  'pixel459',\n",
       "  'pixel460',\n",
       "  'pixel461',\n",
       "  'pixel462',\n",
       "  'pixel463',\n",
       "  'pixel464',\n",
       "  'pixel465',\n",
       "  'pixel466',\n",
       "  'pixel467',\n",
       "  'pixel468',\n",
       "  'pixel469',\n",
       "  'pixel470',\n",
       "  'pixel471',\n",
       "  'pixel472',\n",
       "  'pixel473',\n",
       "  'pixel474',\n",
       "  'pixel475',\n",
       "  'pixel476',\n",
       "  'pixel477',\n",
       "  'pixel478',\n",
       "  'pixel479',\n",
       "  'pixel480',\n",
       "  'pixel481',\n",
       "  'pixel482',\n",
       "  'pixel483',\n",
       "  'pixel484',\n",
       "  'pixel485',\n",
       "  'pixel486',\n",
       "  'pixel487',\n",
       "  'pixel488',\n",
       "  'pixel489',\n",
       "  'pixel490',\n",
       "  'pixel491',\n",
       "  'pixel492',\n",
       "  'pixel493',\n",
       "  'pixel494',\n",
       "  'pixel495',\n",
       "  'pixel496',\n",
       "  'pixel497',\n",
       "  'pixel498',\n",
       "  'pixel499',\n",
       "  'pixel500',\n",
       "  'pixel501',\n",
       "  'pixel502',\n",
       "  'pixel503',\n",
       "  'pixel504',\n",
       "  'pixel505',\n",
       "  'pixel506',\n",
       "  'pixel507',\n",
       "  'pixel508',\n",
       "  'pixel509',\n",
       "  'pixel510',\n",
       "  'pixel511',\n",
       "  'pixel512',\n",
       "  'pixel513',\n",
       "  'pixel514',\n",
       "  'pixel515',\n",
       "  'pixel516',\n",
       "  'pixel517',\n",
       "  'pixel518',\n",
       "  'pixel519',\n",
       "  'pixel520',\n",
       "  'pixel521',\n",
       "  'pixel522',\n",
       "  'pixel523',\n",
       "  'pixel524',\n",
       "  'pixel525',\n",
       "  'pixel526',\n",
       "  'pixel527',\n",
       "  'pixel528',\n",
       "  'pixel529',\n",
       "  'pixel530',\n",
       "  'pixel531',\n",
       "  'pixel532',\n",
       "  'pixel533',\n",
       "  'pixel534',\n",
       "  'pixel535',\n",
       "  'pixel536',\n",
       "  'pixel537',\n",
       "  'pixel538',\n",
       "  'pixel539',\n",
       "  'pixel540',\n",
       "  'pixel541',\n",
       "  'pixel542',\n",
       "  'pixel543',\n",
       "  'pixel544',\n",
       "  'pixel545',\n",
       "  'pixel546',\n",
       "  'pixel547',\n",
       "  'pixel548',\n",
       "  'pixel549',\n",
       "  'pixel550',\n",
       "  'pixel551',\n",
       "  'pixel552',\n",
       "  'pixel553',\n",
       "  'pixel554',\n",
       "  'pixel555',\n",
       "  'pixel556',\n",
       "  'pixel557',\n",
       "  'pixel558',\n",
       "  'pixel559',\n",
       "  'pixel560',\n",
       "  'pixel561',\n",
       "  'pixel562',\n",
       "  'pixel563',\n",
       "  'pixel564',\n",
       "  'pixel565',\n",
       "  'pixel566',\n",
       "  'pixel567',\n",
       "  'pixel568',\n",
       "  'pixel569',\n",
       "  'pixel570',\n",
       "  'pixel571',\n",
       "  'pixel572',\n",
       "  'pixel573',\n",
       "  'pixel574',\n",
       "  'pixel575',\n",
       "  'pixel576',\n",
       "  'pixel577',\n",
       "  'pixel578',\n",
       "  'pixel579',\n",
       "  'pixel580',\n",
       "  'pixel581',\n",
       "  'pixel582',\n",
       "  'pixel583',\n",
       "  'pixel584',\n",
       "  'pixel585',\n",
       "  'pixel586',\n",
       "  'pixel587',\n",
       "  'pixel588',\n",
       "  'pixel589',\n",
       "  'pixel590',\n",
       "  'pixel591',\n",
       "  'pixel592',\n",
       "  'pixel593',\n",
       "  'pixel594',\n",
       "  'pixel595',\n",
       "  'pixel596',\n",
       "  'pixel597',\n",
       "  'pixel598',\n",
       "  'pixel599',\n",
       "  'pixel600',\n",
       "  'pixel601',\n",
       "  'pixel602',\n",
       "  'pixel603',\n",
       "  'pixel604',\n",
       "  'pixel605',\n",
       "  'pixel606',\n",
       "  'pixel607',\n",
       "  'pixel608',\n",
       "  'pixel609',\n",
       "  'pixel610',\n",
       "  'pixel611',\n",
       "  'pixel612',\n",
       "  'pixel613',\n",
       "  'pixel614',\n",
       "  'pixel615',\n",
       "  'pixel616',\n",
       "  'pixel617',\n",
       "  'pixel618',\n",
       "  'pixel619',\n",
       "  'pixel620',\n",
       "  'pixel621',\n",
       "  'pixel622',\n",
       "  'pixel623',\n",
       "  'pixel624',\n",
       "  'pixel625',\n",
       "  'pixel626',\n",
       "  'pixel627',\n",
       "  'pixel628',\n",
       "  'pixel629',\n",
       "  'pixel630',\n",
       "  'pixel631',\n",
       "  'pixel632',\n",
       "  'pixel633',\n",
       "  'pixel634',\n",
       "  'pixel635',\n",
       "  'pixel636',\n",
       "  'pixel637',\n",
       "  'pixel638',\n",
       "  'pixel639',\n",
       "  'pixel640',\n",
       "  'pixel641',\n",
       "  'pixel642',\n",
       "  'pixel643',\n",
       "  'pixel644',\n",
       "  'pixel645',\n",
       "  'pixel646',\n",
       "  'pixel647',\n",
       "  'pixel648',\n",
       "  'pixel649',\n",
       "  'pixel650',\n",
       "  'pixel651',\n",
       "  'pixel652',\n",
       "  'pixel653',\n",
       "  'pixel654',\n",
       "  'pixel655',\n",
       "  'pixel656',\n",
       "  'pixel657',\n",
       "  'pixel658',\n",
       "  'pixel659',\n",
       "  'pixel660',\n",
       "  'pixel661',\n",
       "  'pixel662',\n",
       "  'pixel663',\n",
       "  'pixel664',\n",
       "  'pixel665',\n",
       "  'pixel666',\n",
       "  'pixel667',\n",
       "  'pixel668',\n",
       "  'pixel669',\n",
       "  'pixel670',\n",
       "  'pixel671',\n",
       "  'pixel672',\n",
       "  'pixel673',\n",
       "  'pixel674',\n",
       "  'pixel675',\n",
       "  'pixel676',\n",
       "  'pixel677',\n",
       "  'pixel678',\n",
       "  'pixel679',\n",
       "  'pixel680',\n",
       "  'pixel681',\n",
       "  'pixel682',\n",
       "  'pixel683',\n",
       "  'pixel684',\n",
       "  'pixel685',\n",
       "  'pixel686',\n",
       "  'pixel687',\n",
       "  'pixel688',\n",
       "  'pixel689',\n",
       "  'pixel690',\n",
       "  'pixel691',\n",
       "  'pixel692',\n",
       "  'pixel693',\n",
       "  'pixel694',\n",
       "  'pixel695',\n",
       "  'pixel696',\n",
       "  'pixel697',\n",
       "  'pixel698',\n",
       "  'pixel699',\n",
       "  'pixel700',\n",
       "  'pixel701',\n",
       "  'pixel702',\n",
       "  'pixel703',\n",
       "  'pixel704',\n",
       "  'pixel705',\n",
       "  'pixel706',\n",
       "  'pixel707',\n",
       "  'pixel708',\n",
       "  'pixel709',\n",
       "  'pixel710',\n",
       "  'pixel711',\n",
       "  'pixel712',\n",
       "  'pixel713',\n",
       "  'pixel714',\n",
       "  'pixel715',\n",
       "  'pixel716',\n",
       "  'pixel717',\n",
       "  'pixel718',\n",
       "  'pixel719',\n",
       "  'pixel720',\n",
       "  'pixel721',\n",
       "  'pixel722',\n",
       "  'pixel723',\n",
       "  'pixel724',\n",
       "  'pixel725',\n",
       "  'pixel726',\n",
       "  'pixel727',\n",
       "  'pixel728',\n",
       "  'pixel729',\n",
       "  'pixel730',\n",
       "  'pixel731',\n",
       "  'pixel732',\n",
       "  'pixel733',\n",
       "  'pixel734',\n",
       "  'pixel735',\n",
       "  'pixel736',\n",
       "  'pixel737',\n",
       "  'pixel738',\n",
       "  'pixel739',\n",
       "  'pixel740',\n",
       "  'pixel741',\n",
       "  'pixel742',\n",
       "  'pixel743',\n",
       "  'pixel744',\n",
       "  'pixel745',\n",
       "  'pixel746',\n",
       "  'pixel747',\n",
       "  'pixel748',\n",
       "  'pixel749',\n",
       "  'pixel750',\n",
       "  'pixel751',\n",
       "  'pixel752',\n",
       "  'pixel753',\n",
       "  'pixel754',\n",
       "  'pixel755',\n",
       "  'pixel756',\n",
       "  'pixel757',\n",
       "  'pixel758',\n",
       "  'pixel759',\n",
       "  'pixel760',\n",
       "  'pixel761',\n",
       "  'pixel762',\n",
       "  'pixel763',\n",
       "  'pixel764',\n",
       "  'pixel765',\n",
       "  'pixel766',\n",
       "  'pixel767',\n",
       "  'pixel768',\n",
       "  'pixel769',\n",
       "  'pixel770',\n",
       "  'pixel771',\n",
       "  'pixel772',\n",
       "  'pixel773',\n",
       "  'pixel774',\n",
       "  'pixel775',\n",
       "  'pixel776',\n",
       "  'pixel777',\n",
       "  'pixel778',\n",
       "  'pixel779',\n",
       "  'pixel780',\n",
       "  'pixel781',\n",
       "  'pixel782',\n",
       "  'pixel783',\n",
       "  'pixel784'],\n",
       " 'target_names': ['class'],\n",
       " 'DESCR': \"**Author**: Yann LeCun, Corinna Cortes, Christopher J.C. Burges  \\n**Source**: [MNIST Website](http://yann.lecun.com/exdb/mnist/) - Date unknown  \\n**Please cite**:  \\n\\nThe MNIST database of handwritten digits with 784 features, raw data available at: http://yann.lecun.com/exdb/mnist/. It can be split in a training set of the first 60,000 examples, and a test set of 10,000 examples  \\n\\nIt is a subset of a larger set available from NIST. The digits have been size-normalized and centered in a fixed-size image. It is a good database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting. The original black and white (bilevel) images from NIST were size normalized to fit in a 20x20 pixel box while preserving their aspect ratio. The resulting images contain grey levels as a result of the anti-aliasing technique used by the normalization algorithm. the images were centered in a 28x28 image by computing the center of mass of the pixels, and translating the image so as to position this point at the center of the 28x28 field.  \\n\\nWith some classification methods (particularly template-based methods, such as SVM and K-nearest neighbors), the error rate improves when the digits are centered by bounding box rather than center of mass. If you do this kind of pre-processing, you should report it in your publications. The MNIST database was constructed from NIST's NIST originally designated SD-3 as their training set and SD-1 as their test set. However, SD-3 is much cleaner and easier to recognize than SD-1. The reason for this can be found on the fact that SD-3 was collected among Census Bureau employees, while SD-1 was collected among high-school students. Drawing sensible conclusions from learning experiments requires that the result be independent of the choice of training set and test among the complete set of samples. Therefore it was necessary to build a new database by mixing NIST's datasets.  \\n\\nThe MNIST training set is composed of 30,000 patterns from SD-3 and 30,000 patterns from SD-1. Our test set was composed of 5,000 patterns from SD-3 and 5,000 patterns from SD-1. The 60,000 pattern training set contained examples from approximately 250 writers. We made sure that the sets of writers of the training set and test set were disjoint. SD-1 contains 58,527 digit images written by 500 different writers. In contrast to SD-3, where blocks of data from each writer appeared in sequence, the data in SD-1 is scrambled. Writer identities for SD-1 is available and we used this information to unscramble the writers. We then split SD-1 in two: characters written by the first 250 writers went into our new training set. The remaining 250 writers were placed in our test set. Thus we had two sets with nearly 30,000 examples each. The new training set was completed with enough examples from SD-3, starting at pattern # 0, to make a full set of 60,000 training patterns. Similarly, the new test set was completed with SD-3 examples starting at pattern # 35,000 to make a full set with 60,000 test patterns. Only a subset of 10,000 test images (5,000 from SD-1 and 5,000 from SD-3) is available on this site. The full 60,000 sample training set is available.\\n\\nDownloaded from openml.org.\",\n",
       " 'details': {'id': '554',\n",
       "  'name': 'mnist_784',\n",
       "  'version': '1',\n",
       "  'format': 'ARFF',\n",
       "  'upload_date': '2014-09-29T03:28:38',\n",
       "  'licence': 'Public',\n",
       "  'url': 'https://www.openml.org/data/v1/download/52667/mnist_784.arff',\n",
       "  'file_id': '52667',\n",
       "  'default_target_attribute': 'class',\n",
       "  'tag': ['AzurePilot',\n",
       "   'OpenML-CC18',\n",
       "   'OpenML100',\n",
       "   'study_1',\n",
       "   'study_123',\n",
       "   'study_41',\n",
       "   'study_99',\n",
       "   'vision'],\n",
       "  'visibility': 'public',\n",
       "  'status': 'active',\n",
       "  'processing_date': '2018-10-03 21:23:30',\n",
       "  'md5_checksum': '0298d579eb1b86163de7723944c7e495'},\n",
       " 'url': 'https://www.openml.org/d/554'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70000, 784)\n",
      "(70000,)\n"
     ]
    }
   ],
   "source": [
    "# 数据和标签\n",
    "X, y = mnist['data'], mnist['target']\n",
    "print(X.shape)\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(784,)\n",
      "5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOZ0lEQVR4nO3dbYxc5XnG8euKbezamMQbB9chLjjgFAg0Jl0ZEBZQoVCCKgGqArGiyKG0ThOchNaVoLQqtKKVWyVElFIkU1xMxUsgAeEPNAm1ECRqcFlcY2wIb8Y0NmaNWYENIX5Z3/2w42iBnWeXmTMv3vv/k1Yzc+45c24NXD5nznNmHkeEAIx/H+p0AwDag7ADSRB2IAnCDiRB2IEkJrZzY4d5ckzRtHZuEkjlV3pbe2OPR6o1FXbb50m6QdIESf8WEctLz5+iaTrV5zSzSQAFa2NN3VrDh/G2J0i6SdLnJZ0oaZHtExt9PQCt1cxn9gWSXoiIzRGxV9Ldki6opi0AVWsm7EdJ+sWwx1try97F9hLbfbb79mlPE5sD0IyWn42PiBUR0RsRvZM0udWbA1BHM2HfJmnOsMefqC0D0IWaCfvjkubZnmv7MElflLS6mrYAVK3hobeI2G97qaQfaWjobWVEbKqsMwCVamqcPSIelPRgRb0AaCEulwWSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJpmZxRffzxPJ/4gkfm9nS7T/7F8fUrQ1OPVBc9+hjdxTrU7/uYv3V6w+rW1vX+73iujsH3y7WT713WbF+3J8/Vqx3QlNht71F0m5Jg5L2R0RvFU0BqF4Ve/bfi4idFbwOgBbiMzuQRLNhD0k/tv2E7SUjPcH2Ett9tvv2aU+TmwPQqGYP4xdGxDbbR0p6yPbPI+LR4U+IiBWSVkjSEe6JJrcHoEFN7dkjYlvtdoek+yUtqKIpANVrOOy2p9mefvC+pHMlbayqMQDVauYwfpak+20ffJ07I+KHlXQ1zkw4YV6xHpMnFeuvnPWRYv2d0+qPCfd8uDxe/JPPlMebO+k/fzm9WP/HfzmvWF978p11ay/te6e47vL+zxXrH//JofeJtOGwR8RmSZ+psBcALcTQG5AEYQeSIOxAEoQdSIKwA0nwFdcKDJ792WL9+ttuKtY/Nan+VzHHs30xWKz/zY1fKdYnvl0e/jr93qV1a9O37S+uO3lneWhuat/aYr0bsWcHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQYZ6/A5GdfKdaf+NWcYv1Tk/qrbKdSy7afVqxvfqv8U9S3Hfv9urU3D5THyWf9838X66106H2BdXTs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCUe0b0TxCPfEqT6nbdvrFgOXnl6s7zqv/HPPEzYcXqw/+fUbP3BPB12383eK9cfPKo+jD77xZrEep9f/AeIt3yyuqrmLniw/Ae+zNtZoVwyMOJc1e3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9i4wYeZHi/XB1weK9ZfurD9WvunMlcV1F/zDN4r1I2/q3HfK8cE1Nc5ue6XtHbY3DlvWY/sh28/XbmdU2TCA6o3lMP42Se+d9f4qSWsiYp6kNbXHALrYqGGPiEclvfc48gJJq2r3V0m6sNq2AFSt0d+gmxUR22v3X5U0q94TbS+RtESSpmhqg5sD0Kymz8bH0Bm+umf5ImJFRPRGRO8kTW52cwAa1GjY+23PlqTa7Y7qWgLQCo2GfbWkxbX7iyU9UE07AFpl1M/stu+SdLakmba3SrpG0nJJ99i+TNLLki5uZZPj3eDO15taf9+uxud3//SXni7WX7t5QvkFDpTnWEf3GDXsEbGoTomrY4BDCJfLAkkQdiAJwg4kQdiBJAg7kARTNo8DJ1z5XN3apSeXB03+/eg1xfpZX7i8WJ/+vceKdXQP9uxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7ONAadrk1792QnHd/1v9TrF+1XW3F+t/efFFxXr874fr1ub8/c+K66qNP3OeAXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCKZuTG/ij04v1O675drE+d+KUhrf96duXFuvzbtlerO/fvKXhbY9XTU3ZDGB8IOxAEoQdSIKwA0kQdiAJwg4kQdiBJBhnR1GcMb9YP2L51mL9rk/+qOFtH//wHxfrv/239b/HL0mDz29ueNuHqqbG2W2vtL3D9sZhy661vc32+trf+VU2DKB6YzmMv03SeSMs/25EzK/9PVhtWwCqNmrYI+JRSQNt6AVACzVzgm6p7Q21w/wZ9Z5ke4ntPtt9+7Snic0BaEajYb9Z0rGS5kvaLuk79Z4YESsiojcieidpcoObA9CshsIeEf0RMRgRByTdImlBtW0BqFpDYbc9e9jDiyRtrPdcAN1h1HF223dJOlvSTEn9kq6pPZ4vKSRtkfTViCh/+ViMs49HE2YdWay/cslxdWtrr7yhuO6HRtkXfemlc4v1Nxe+XqyPR6Vx9lEniYiIRSMsvrXprgC0FZfLAkkQdiAJwg4kQdiBJAg7kARfcUXH3LO1PGXzVB9WrP8y9hbrf/CNK+q/9v1ri+seqvgpaQCEHciCsANJEHYgCcIOJEHYgSQIO5DEqN96Q24HFs4v1l/8QnnK5pPmb6lbG20cfTQ3DpxSrE99oK+p1x9v2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs49z7j2pWH/um+Wx7lvOWFWsnzml/J3yZuyJfcX6YwNzyy9wYNRfN0+FPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4+yFg4tyji/UXL/143dq1l9xdXPcPD9/ZUE9VuLq/t1h/5IbTivUZq8q/O493G3XPbnuO7YdtP217k+1v1Zb32H7I9vO12xmtbxdAo8ZyGL9f0rKIOFHSaZIut32ipKskrYmIeZLW1B4D6FKjhj0itkfEutr93ZKekXSUpAskHbyWcpWkC1vUI4AKfKDP7LaPkXSKpLWSZkXEwYuPX5U0q846SyQtkaQpmtpwowCaM+az8bYPl/QDSVdExK7htRiaHXLEGSIjYkVE9EZE7yRNbqpZAI0bU9htT9JQ0O+IiPtqi/ttz67VZ0va0ZoWAVRh1MN425Z0q6RnIuL6YaXVkhZLWl67faAlHY4DE4/5rWL9zd+dXaxf8nc/LNb/9CP3FeuttGx7eXjsZ/9af3it57b/Ka474wBDa1Uay2f2MyR9WdJTttfXll2toZDfY/sySS9LurglHQKoxKhhj4ifShpxcndJ51TbDoBW4XJZIAnCDiRB2IEkCDuQBGEHkuArrmM0cfZv1q0NrJxWXPdrcx8p1hdN72+opyos3bawWF938/xifeb3NxbrPbsZK+8W7NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IIk04+x7f7/8s8V7/2ygWL/6uAfr1s79jbcb6qkq/YPv1K2duXpZcd3j//rnxXrPG+Vx8gPFKroJe3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSCLNOPuWC8v/rj138r0t2/ZNbxxbrN/wyLnFugfr/bjvkOOve6lubV7/2uK6g8UqxhP27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQhCOi/AR7jqTbJc2SFJJWRMQNtq+V9CeSXqs99eqIqP+lb0lHuCdONRO/Aq2yNtZoVwyMeGHGWC6q2S9pWUSssz1d0hO2H6rVvhsR366qUQCtM5b52bdL2l67v9v2M5KOanVjAKr1gT6z2z5G0imSDl6DudT2Btsrbc+os84S2322+/ZpT3PdAmjYmMNu+3BJP5B0RUTsknSzpGMlzdfQnv87I60XESsiojcieidpcvMdA2jImMJue5KGgn5HRNwnSRHRHxGDEXFA0i2SFrSuTQDNGjXsti3pVknPRMT1w5bPHva0iySVp/ME0FFjORt/hqQvS3rK9vrasqslLbI9X0PDcVskfbUF/QGoyFjOxv9U0kjjdsUxdQDdhSvogCQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSYz6U9KVbsx+TdLLwxbNlLSzbQ18MN3aW7f2JdFbo6rs7eiI+NhIhbaG/X0bt/siordjDRR0a2/d2pdEb41qV28cxgNJEHYgiU6HfUWHt1/Srb11a18SvTWqLb119DM7gPbp9J4dQJsQdiCJjoTd9nm2n7X9gu2rOtFDPba32H7K9nrbfR3uZaXtHbY3DlvWY/sh28/XbkecY69DvV1re1vtvVtv+/wO9TbH9sO2n7a9yfa3ass7+t4V+mrL+9b2z+y2J0h6TtLnJG2V9LikRRHxdFsbqcP2Fkm9EdHxCzBsnynpLUm3R8RJtWX/JGkgIpbX/qGcERFXdklv10p6q9PTeNdmK5o9fJpxSRdK+oo6+N4V+rpYbXjfOrFnXyDphYjYHBF7Jd0t6YIO9NH1IuJRSQPvWXyBpFW1+6s09D9L29XprStExPaIWFe7v1vSwWnGO/reFfpqi06E/ShJvxj2eKu6a773kPRj20/YXtLpZkYwKyK21+6/KmlWJ5sZwajTeLfTe6YZ75r3rpHpz5vFCbr3WxgRn5X0eUmX1w5Xu1IMfQbrprHTMU3j3S4jTDP+a5187xqd/rxZnQj7Nklzhj3+RG1ZV4iIbbXbHZLuV/dNRd1/cAbd2u2ODvfza900jfdI04yrC967Tk5/3omwPy5pnu25tg+T9EVJqzvQx/vYnlY7cSLb0ySdq+6binq1pMW1+4slPdDBXt6lW6bxrjfNuDr83nV8+vOIaPufpPM1dEb+RUl/1Yke6vT1SUlP1v42dbo3SXdp6LBun4bObVwm6aOS1kh6XtJ/Serpot7+Q9JTkjZoKFizO9TbQg0dom+QtL72d36n37tCX21537hcFkiCE3RAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kMT/A65XcTMQuIbWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "some_digit = X[0]\n",
    "print(some_digit.shape)\n",
    "img_some_digit = some_digit.reshape(28,28)\n",
    "plt.imshow(img_some_digit)\n",
    "print(y[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建立测试集和训练集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 784)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 随机打乱数据\n",
    "import numpy as np\n",
    "shuffle_index = np.random.permutation(60000)\n",
    "X_train, y_train = X_train[shuffle_index], y_train[shuffle_index]\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['4', '4', '6', ..., '0', '4', '6'], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[0] == '5'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 二元分类器\n",
    "\n",
    "识别数字5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "先对标签进行处理，将它处理为5和非5两种形式的标签\n",
    "注意：这里的5应该是'5'而不是数字5，否则二元分类器会训练错误"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ..., False, False, False])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_5 = (y_train == '5')\n",
    "# y_train_5.shape\n",
    "y_train_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000,)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_5= (y_test == '5')\n",
    "y_test_5.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(random_state=42)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "sgd_clf = SGDClassifier(random_state = 42)\n",
    "sgd_clf.fit(X_train, y_train_5)\n",
    "# sgd_clf.predict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ..., False,  True, False])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 性能考核"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1、交叉验证测量精度值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.9667 , 0.96495, 0.90455])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "cross_val_score(sgd_clf, X_train, y_train_5, cv = 3, scoring = \"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_predict\n",
    "\n",
    "y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_5, cv = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ...,  True, False, False])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "cross_val_predict 与 cross_val_score相比，都是同样执行交叉验证，但它返回的是每次交叉折叠的预测结果，而不是评估分数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2、混淆矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[52298,  2281],\n",
       "       [  995,  4426]], dtype=int64)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3、精度和召回率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6599075592664381\n",
      "0.8164545286847446\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "print(precision_score(y_train_5, y_train_pred))\n",
    "print(recall_score(y_train_5, y_train_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4、F1 score\n",
    "\n",
    "说明:检测一张图的时候，只有90%的概率是准确的，而且只有64%的数字5 被它检测出来\n",
    "\n",
    "精度和召回率合成单一指标，成为 F1 分数，谐波平均值\n",
    "\n",
    "平均值平等对待所有的值，谐波平均值会给予较低值更高的权重，只有召回率和精度都很高时，才能获得较高的F1分数\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7298812664907651"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "f1_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5、精度与召回如何做权衡？\n",
    "\n",
    "F1分数对那些具有相近精度和召回率 分类器更有利，这不一定符合你的期望\n",
    "\n",
    "有时候你更关心精度，有时你能关心召回率\n",
    "\n",
    "比如：训练一个分类器检测儿童可以放心观看的视频，你可能要求拦截了很多好的视频，低召回率，保留下来的都是安全的视频，高精度\n",
    "\n",
    "不能同时增加精度并减少召回率，反之亦然"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sklearn不可以直接设置阈值，但可以访问决策分数进行设置。\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2771.22911393])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 返回决策值decision_function\n",
    "y_scores = sgd_clf.decision_function([some_digit])\n",
    "y_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold = 0\n",
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_scores = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3,\n",
    "                             method=\"decision_function\")   # 返回决策值，而不是预测结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_scores.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "6、PR曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAEPCAYAAABx8azBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAr8UlEQVR4nO3deZwU1b338c+P2Rlm2EFgkEVQFFRAMIjro4iICXo1Roh7ch+8IZoniTcukRgV4xKXuESiJCHEGDUgELleE5dETaKojHFDFh0RZREZFtmGWZo5zx9VPdMz9GzQTNV0fd+vV7+66tTp7l/1Mr85p06dMuccIiIiEpx2QQcgIiISdUrGIiIiAVMyFhERCZiSsYiISMCUjEVERAKmZCwiIhKwJpOxmc02s41mtrSB7WZmD5hZiZm9Z2YjUx+miIhI+mpOy3gOMKGR7WcCg/3bVOBX+x+WiIhIdDSZjJ1z/wC2NFLlbOBR53kd6GRmvVIVoIiISLrLTMFz9AHWJKyv9cs+r1/RzKbitZ7Jz88/ZsiQISl4eZFgVFVX8d4X70HAk9jlZuVSkF1AYU4hhTmFtLP9Hwry+eewfn3ybYcdBh067PdLiETOW2+9tck51z3ZtlQk42Zzzs0CZgGMGjXKFRcXt+bLi6RUtavmtTWvsax0Gc45HI749LLx5YbKgDrbm1NWEatg8+7NbCrbRHmsnLXb1/LOhncod+WUU04ppQAc3/d4Lh9+Od8a8S3MrMX75Rxccw3cfTdMmQITJ9bdPn489OixT2+ZSKSZ2acNbUtFMl4H9E1YL/LLRNJaO2vHCQefwAkHnxBYDJV7Knlx1Yv8teSv/O9H/8uqrat4dc2rvLrmVX7z9m+Yd/48igqLmvVcxcVQUQEn+Lszdy6ccw5kZR24+EXEk4pTmxYBl/ijqscA25xze3VRi0jqZWdkM3HwRB448wE+/t7HfH7159wz/h7ys/J5fe3rHDPrGL7Y+UWTz/PqqzB6dG0iBnj/fSVikdbSnFObngAWA4eZ2Voz+7aZ/ZeZ/Zdf5VlgFVAC/BqYdsCiFZFGHdThIH543A957zvvcWjXQ9m4ayO3vHJLk487IUnj/uqrD0CAIpKUBXUJRR0zFjmw3v78bY6ZdQwZ7TJY+p2lHNbtsL3qvPEGjBmz92MHDoSPP26FIEUixMzecs6NSrZNM3CJpKkRvUYwedhkYtUx7n7t7qR16ifiBx6ANWvgrbdaIUARqaFkLJLGbjjxBgAeX/o45bHyOttWr65b9+ab4aqroKgIOnVqnfhExNOqpza11LZt29i0aROVlZVBhyKtKDs7m27dutGxY8egQ2nzhvYYyvCDhvPOhnd4ZfUrnDHojJpts2fX1gvoaJWI+EKbjMvLy/niiy8oKioiLy9vn86XlLbHOcfu3btZu3YtOTk55ObmBh1Sm3d83+N5Z8M7vL3h7TrJeMaMAIMSkTpC201dWlpK9+7dad++vRJxhJgZ7du3p1u3bpSWlgYdTlo4vu/xAPztk7/VKb//fu/+yitbOyIRqS+0ybi8vJwOmnMvsgoKCigvL2+6ojTp1AGnAvDqZ68Sq47VlH/ve1739IMPBhWZiMSFNhnHYjEyM0Pbiy4HWGZmJrFYrOmK0qSeHXpSVFjE7thuVn+5uqa8Wzc49NDg4hKRWqFNxoC6pyNMn31qDe4yGICSLSUAbN0KmzfDRx8FGZWIxIU6GYtIavTv1B+AT7/05ql/4YUAgxGRvSgZi0RAv479APhs22cArFgRZDQiUp+ScSuaM2cOZlZzKygo4Oijj+aXv/xlqx0fXb16NWbGnDlzmv2YeNyr688SIW1Gv05eMl69bTXgdVMDHHFEQAGJSB0aIRWAefPmUVRUxPbt25k3bx5XXXUVGzdu5JZbmp7Qf3/16tWLxYsXc8ghhzT7MWeddRaLFy+mV69eBzAyOZDqt4xfftkr79kzoIBEpA4l4wAMHz6cQYMGATB+/HhKSkq4//77kybjqqoqMjMzUzagKScnhzHJrgzQiO7du9O9e/eUvL4Eo09hHwDW71gPwDvveOVHHRVQQCJSh7qpQ2D06NFs376dN998EzNj5syZXHPNNfTu3ZucnBy+/PJLABYsWMCYMWNo3749nTp14vzzz+ezzz7b6/l+/etfM3LkSPLy8ujcuTMnn3wyr732GpC8m3rJkiWcfvrpdO3alby8PAYOHMi0abVXwkzWTV1VVcX06dPp378/2dnZ9O/fn+nTp1NVVVVTJ/5ajzzyCDfeeCO9evWiU6dOfO1rX2Pt2rWpfROlUX0KvGS8dvtaEq/UplObRMKhzSVjs4Zvs2bV1ps1q/G6iY45puF6U6fW1jtQV7L55JNPyMjIqJnk5Gc/+xkffvghs2bNYuHCheTm5vLwww9z3nnnccQRR/DUU0/xyCOPsHTpUk4++WR27NhR81z//d//zdSpUxk5ciRz587lscce46STTkqatAF27tzJGWecQUZGBnPmzOEvf/kLN954Y5PHsC+99FLuuOMOLrnkEp555hkuu+wy7rzzTi699NK96t5+++2UlJQwe/Zs7r//fhYvXsxFF120H++YtFR+dj4dczpSuaeSzbs3s2ePd2rTNF19XCQU1E0dgD179hCLxdixYwdz585lwYIFfO1rX6N9+/YA9OzZk4ULF9Z0Te/cuZNrr72Wyy+/nNkJs/sfe+yxHHbYYfz2t7/l+9//PiUlJfziF7/gBz/4Affee29NvbPOOqvBWFasWMHWrVv5+c9/zlEJfZaXXXZZg49ZunQpTzzxBD/96U+56aabAK+7PTMzk5/85Cdcd911dZ6rf//+PP744zXrpaWl/OhHP2L9+vX07t27eW+a7LeDOhzEtoptbNy1kW7tu9GlS9ARiUhcm2sZO9fwLbEVO3Vq43UTvfVWw/USW9vHHJOafRgyZAhZWVl06dKFadOmceGFF9ZJsuecc06dY8SLFy9m+/btXHjhhcRisZpb3759GTJkCP/4xz8AePHFF6murmZq4hvRhMGDB9OpUyeuuOIKHnvsMdasWdPkY+KvV791G19/5ZVX6pRPnDixzvqRRx4J0GBrXQ6Mbu27AVC6q5Tq6oCDEZE62lwyTgcLFy5kyZIlrFixgl27dvHoo4/SJaGZUn/U8saNGwEYN24cWVlZdW7vv/8+mzdvBqi5LyoqanYsHTt25KWXXqJ3795MmzaNgw8+mGHDhjF//vwGH7Nly5akcR500EF1tsd1qdcEy8nJAdDc060snow3lW0mJwcKCkAzjoqEg7qpAzBs2LCa0dTJ1B853bVrV8AbSDV06NC96hcUFADQrZv3x3bdunUcdthhzY5n+PDhzJ8/n1gsRnFxMbfffjvf+MY3ePfddxk2bNhe9ePJdcOGDXVOkdqwYUOd7RIunfM6A7Bh21ZiMdi5EzT9u0g4qGXcBowdO5aCggJKSkoYNWrUXrd44h03bhzt2rVjVmLfegtkZmYyZswYZsyYQXV1NcuXL09a76STTgLgySefrFP+xz/+EYBTTjlln15fDqwuud4/SWs2bWmipoi0Nv1f3AYUFhZy11138d3vfpfS0lLOPPNMOnbsyLp163jllVc45ZRT+OY3v8khhxxSM3hrx44dTJo0iYyMDN58802GDBnCBRdcsNdzP/PMM8yaNYtzzjmHAQMGsGvXLh544AEKCgo47rjjksYzbNgwpkyZwk033UQsFmPs2LEsXryYGTNmMGXKlJpjwhIu3fO9c8U3bN8YcCQiUp+ScRtxxRVX0LdvX+666y4ef/xxYrEYffr04cQTT2T48OE19e6++24GDRrEzJkz+f3vf09+fj5HHXUU48ePT/q8gwcPJi8vjxkzZvD5559TUFDA6NGjeeGFFxo99jxnzhwGDhzI7NmzufXWW+nduzfXXnstP/3pT1O965Ii8WPG67dtDjgSEanPXP2hxa1k1KhRrri4uMHty5cv5/DDD2/FiCRs9B1IrfnL5vP1eV/nkMpz+Pi2hcDeZxaIyIFjZm8550Yl26ZjxiIR0TG3IwB5nbYDukiESJiom1okIgpzCgHI7LCNf/8b8vMDDkhEaigZi0RExxyvZbwzto0RIwIORkTqUDe1SETEu6m3V2wPOBIRqU/JWCQi4t3UW3ZtwwwefTTggESkhpKxSETkZeaRYRnEqIB2Vdx2W9ARiUickrFIRJgZ+dn+qK3sXUyeHGw8IlJLyVgkQvKz/GSctQv/uh4iEgJKxiIRUtsy3ol/+WwRCQElY5EI6ZDdwVvI3kleXrCxiEgtJeNWNGfOHMys5padnc0hhxzCj3/840Cv7XvZZZfRv3//mvXVq1djZsyZMyewmOTAqE3Gu9QyFgkRTfoRgHnz5lFUVMSOHTtYuHAht99+Ozt27ODBBx8MOjRJc/FjxocfvYsePQIORkRqNKtlbGYTzGylmZWY2XVJth9sZi+Z2dtm9p6ZTUx9qOlj+PDhjBkzhtNPP52ZM2cybtw4Zs+eTXV1ddChSZqLt4xvuX0Xo0cHHIyI1GgyGZtZBvAQcCZwBDDFzOpPMT8dmOucGwFMBmamOtB0NnLkSMrKyti0aRMAZWVlXHvttQwYMIDs7GwGDBjAz372s72SdWlpKdOmTaNv377k5OTQt29fLr74YioqKgAoKSnh4osvZsCAAeTl5TFw4EC+853vsHXr1lbfRwmH+ACunZU7A45ERBI1p5v6WKDEObcKwMyeBM4GliXUcUChv9wRWJ/KIOPsZjsQT9ti7qepve7c6tWr6dixI127diUWi3HGGWewbNkyfvKTn3DkkUfy+uuvM2PGDLZs2cI999wDwNatWxk7dixbtmxh+vTpHHXUUWzcuJGnn36ayspKcnJyWL9+PX379uW+++6jc+fOrFq1ittuu42JEyeyePHilO6DtA0dsryW8dZdO3EOLBw/KZHIa04y7gOsSVhfC3ylXp2bgOfN7CogHxiX7InMbCowFeDggw9uaaxpY8+ePcRisZpjxvPnz+e+++4jIyODP/zhD/zrX//ilVde4aSTTgLgtNNOA+Dmm2/m2muvpUePHvziF79g1apVFBcXMyJh1v8pU6bULJ900kk1zwEwduxYBg0axIknnsjbb79d53ESDfFu6h9eu4vzn4SiooADEhEgdQO4pgBznHP3mNlxwB/MbJhzrk6/qnNuFjALYNSoUS1uXqa6RRqUIUOG1FmfNm0aV155JQB//etf6devH2PHjiUWi9XUGT9+PNOnT+f1119n0qRJPP/884wePbrRhFpZWcndd9/No48+yqefflpnxPbKlSuVjCMo8dSm3NxgYxGRWs1JxuuAvgnrRX5Zom8DEwCcc4vNLBfoBmxMRZDpZuHChRQVFVFaWsq9997LzJkz+cpXvsIll1zCxo0b+fTTT8nKykr62M2bN9fcH3300Y2+zvXXX8+DDz7IjTfeyNixYykoKGDt2rWce+65gZ5KJcFJTMY5OcHGIiK1mpOMlwCDzWwAXhKeDHyzXp3PgNOAOWZ2OJALlKYy0HQybNgwBg0aBMCpp57KUUcdxY9+9CPOO+88unbtyoABA5g7d27Sx8bPB+7WrRvr1tX/n6iuJ598kksuuYTp06fXlO3cqYE7UVaTjHN2KBmLhEiTydg5FzOzK4HngAxgtnPuAzO7BSh2zi0CrgZ+bWY/wBvMdZlzLj36lA+wnJwc7rrrLs4++2xmzpzJhAkTmD9/Ph06dNirOzvR+PHjufXWW3n33XcbbCGXlZXt1cL+3e9+l9L4pW3Jy6ydm7qBzhcRCUCzjhk7554Fnq1XdmPC8jLg+NSGFh2TJk1i9OjR3HPPPXz00Uf87ne/47TTTuPqq6/m6KOPprKyko8//phFixbx5z//mfbt2/ODH/yAxx9/nHHjxjF9+nSOPPJINm3axNNPP83DDz9MQUEBEyZM4Pe//z1HHnkkgwYNYsGCBbz22mtB764EKMu8abfa5ZRpJLVIiGgGrpC49dZbOeOMM/jNb37Dc889xx133MGsWbP45JNPyM/P55BDDuGss84iOzsbgE6dOvHqq68yffp07rjjDjZv3kzPnj059dRTa+o8+OCDOOe44YYbAJg4cSJPPPEExx57bGD7KcHKcl4ytpyygCMRkUQWVG/yqFGjXHFxcYPbly9fzuGHH96KEUnY6DuQen8v+Ren/fFEDs0dy8prXw06HJFIMbO3nHOjkm3ThSJEIqSwvXc+U4fOGk0vEiZKxiIRkpfpXTdxd9XugCMRkURKxiIRsmub1zLe9KVaxiJhomQsEiFbNnot46071TIWCZNQJ2Odqhxd+uwPkJjXMnYZahmLhElok3FWVha7d+u/96javXt3g1OCyn6o8lrG1Rn6bYmESWiTcY8ePVi3bh1lZWVqJUWIc46ysjLWrVtHjx49gg4n7VRXeXNgunYV+l2JhEhoJ/0oLPQuj7x+/XqqqqoCjkZaU1ZWFj179qz5DkjqVFa0g1gOZFZQHisnLysv6JBEhBAnY/ASsv4gi6RORQXecWMlY5FQCW03tYgcGBbzzzWO6bixSFgoGYtEyAUXQL8ib0R1eUwjqkXCQslYJGI0C5dI+CgZi0RMbqZaxiJho2QsEiEPPQTL39MxY5GwUTIWiZCNG6F8p1rGImGjZCwSId6pTTpmLBI2SsYiEVJznjFqGYuEiZKxSIRUVFAzP7WOGYuEh5KxSISoZSwSTkrGIhFSXo6OGYuEkJKxSIScdRaMGu61jNVNLRIeSsYiEfLNb8LE072WsbqpRcJDyVgkYuIzcKmbWiQ8lIxFImTJElj3qVrGImET6usZi0hqfe978HosF76qY8YiYaJkLBIh5eWAU8tYJGyUjEUipKICaKfR1CJho2PGIhGSODe1WsYi4aGWsUiEVFQAptHUImGjZCwSIRUVQIZaxiJho25qkQhJnJtax4xFwkPJWCRC3n8fFi1Qy1gkbJqVjM1sgpmtNLMSM7uugTrfMLNlZvaBmT2e2jBFJBX69YPDB+uYsUjYNHnM2MwygIeA04G1wBIzW+ScW5ZQZzBwPXC8c26rmfU4UAGLyP7Jy1TLWCRsmjOA61igxDm3CsDMngTOBpYl1Pm/wEPOua0AzrmNqQ5URPZPWRlcfDHkdcmDIh0zFgmT5nRT9wHWJKyv9csSHQocamavmtnrZjYh2ROZ2VQzKzaz4tLS0n2LWET2ya5dsGAB/OV/vG5qtYxFwiNVA7gygcHAKcAU4Ndm1ql+JefcLOfcKOfcqO7du6fopUWkOSoqvPucjNpk7JwLMCIRiWtOMl4H9E1YL/LLEq0FFjnnqpxznwAf4iVnEQmJeDLOy21Hdka2V7anIsCIRCSuOcl4CTDYzAaYWTYwGVhUr86f8VrFmFk3vG7rVakLU0T2V03LOKd2EJdGVIuEQ5PJ2DkXA64EngOWA3Odcx+Y2S1mNsmv9hyw2cyWAS8BP3LObT5QQYtIyyUm49xMHTcWCZNmTYfpnHsWeLZe2Y0Jyw74oX8TkRCq0zLO8lvGGlEtEgqagUskIvLzYdw4OPZYtYxFwkYXihCJiCOPhBde8JZHPqJjxiJhopaxSASpZSwSLmoZi0REebk3C1f79jpmLBI2ahmLRMRTT0HXrvDtb6tlLBI2SsYiEaHzjEXCS8lYJCJ0nrFIeOmYsUhEJCbjWKaOGYuEiZKxSEQkJuM9ahmLhIqSsUhEJCZjl6VjxiJhomQsEhGJyRi1jEVCRclYJCIuvBBGjIChQ+HpzTpmLBImSsYiETF0qHcDeG6xWsYiYaJTm0QiKE/HjEVCRS1jkYhYuBBWroRJkxLOM96jlrFIGCgZi0TEE0/AvHnQvz/kDVPLWCRMlIxFIiI+mjo3F0yjqUVCRclYJCIST23K0FWbREJFyVgkIsr9RnBuLmSoZSwSKkrGIhGR2DLO0lWbREJFyVgkIhKTcbZaxiKhomQsEhH5+dCxI+TlQbaOGYuEipKxSET885+1y2u3q2UsEiaagUskgvL8Y8ZlVWUBRyIioGQsEkn52fkA7KrcFXAkIgJKxiKRMXAgDBoElZWQk5FDO2tHVXUVVXuqgg5NJPKUjEUiYM8e+OQTWLUKsrLAzMjP8lvHVWodiwRNyVgkAnb7g6bz8sDMW453Veu4sUjwlIxFIiCejNu3ry2Lt4x3Vu4MICIRSaRkLBIBZX7jNy+vtix+TeOKWEUAEYlIIiVjkQiIJ+PElnGuZuESCQ0lY5EISDxmHBdPxhV71DIWCZpm4BKJgB49YPp07z4uJyMHUMtYJAyUjEUioKgIZsyoW6ZuapHwUDe1SETlZHotYw3gEgles5KxmU0ws5VmVmJm1zVS7zwzc2Y2KnUhisj+WrMGnn0WPvigtkzd1CLh0WQyNrMM4CHgTOAIYIqZHZGkXgHw/4A3Uh2kiOyfF1+Es86Cu+6qLatpGWsAl0jgmtMyPhYocc6tcs5VAk8CZyepNwO4E9C/2SIhk/TUpgwdMxYJi+Yk4z7AmoT1tX5ZDTMbCfR1zv1vY09kZlPNrNjMiktLS1scrIjsm2QzcGkAl0h47PcALjNrB9wLXN1UXefcLOfcKOfcqO7du+/vS4tIMzU2A9fuqt0BRCQiiZqTjNcBfRPWi/yyuAJgGPCyma0GxgCLNIhLJDwaaxnvjikZiwStOcl4CTDYzAaYWTYwGVgU3+ic2+ac6+ac6++c6w+8DkxyzhUfkIhFpMWStowzvRV1U4sEr8lk7JyLAVcCzwHLgbnOuQ/M7BYzm3SgAxSR/ZesZRzvplYyFgles2bgcs49Czxbr+zGBuqesv9hiUgq3X23Nx1mp061ZfGWsY4ZiwRP02GKREBhoXdLFG8Zl8XKAohIRBJpOkyRiGqf5fVZl1UpGYsETclYJAKuuQbOPReWL68tq7mEouamFgmckrFIBLz8MixcCNu315bp1CaR8FAyFomAZNNhagCXSHgoGYtEQGOnNqllLBI8JWORCGhs0g+1jEWCp2QsEgFqGYuEm5KxSASoZSwSbpr0QyTNOQdnngnl5ZCdXVuulrFIeCgZi6Q5M3j66b3LE69n7JzDzFo5MhGJUze1SES1s3bkZOQAuliESNCUjEXSXFUVrFkDW7fuvU1d1SLhoGQskuY++ggOPhjGjt17W3wQl+anFgmWkrFImouf1pQ4kjquIKcAgJ2VO1sxIhGpT8lYJM0lO60priDbS8Y7Kna0YkQiUp+SsUiaSzbhR1xhjneR4+0V2/feKCKtRslYJM012jL2u6l3VKplLBIkJWORNNdYy7hDdgdAx4xFgqZkLJLmdMxYJPw0A5dImjvjDHjmGejVa+9tNclY3dQigVIyFklzRUXeLRl1U4uEg7qpRSKsZgCXuqlFAqWWsUiamzcPli6F//gPGD687ja1jEXCQS1jkTS3YAHccgssW7b3Nh0zFgkHJWORNLdtm3ffsePe23SesUg4KBmLpLkvv/TuO3Xae5u6qUXCQclYJM3Fk3HSlrHOMxYJBSVjkTQX76ZO1jJWN7VIOCgZi6Q5dVOLhJ9ObRJJY3v2QJ8+sGMH5OfvvV3d1CLhoGQsksYyMuDDDxvenp/tZehdVbuodtW0M3WWiQRBvzyRCGtn7cjP8hNy5a6AoxGJLiVjkYjTIC6R4DUrGZvZBDNbaWYlZnZdku0/NLNlZvaemf3NzPqlPlQRaannn4fCQrjooobr6LixSPCaTMZmlgE8BJwJHAFMMbMj6lV7GxjlnDsKeAr4eaoDFZGW27TJG7wVizVcRyOqRYLXnJbxsUCJc26Vc64SeBI4O7GCc+4l55x/CXNeBxq4YJuItKZNm7z7bt0arqNuapHgNScZ9wHWJKyv9csa8m3gL8k2mNlUMys2s+LS0tLmRyki+yT+M2s0GaubWiRwKR3AZWYXAaOAu5Jtd87Ncs6Ncs6N6t69eypfWkSS+Pxz775Xr4brqJtaJHjNOc94HdA3Yb3IL6vDzMYBNwAnO+cqUhOeiOyP9eu9+969G66jyyiKBK85LeMlwGAzG2Bm2cBkYFFiBTMbATwCTHLObUx9mCKyL+LJuDktY3VTiwSnyZaxcy5mZlcCzwEZwGzn3AdmdgtQ7JxbhNct3QGYZ2YAnznnJh3AuEWkGa67DkpKYODAhuvEB3Cpm1okOM2aDtM59yzwbL2yGxOWx6U4LhFJgcmTm66jbmqR4GkGLpGIUze1SPCUjEXS1IoVMHMmFBc3Xq9zXmcAtpRvaYWoRCQZJWORNPXii/Dd78IjjzRer0d+DwBKd+ncf5GgKBmLpKlly7z7ww9vvF7P/J4AbNi54QBHJCINUTIWSVPvv+/dDx3aeL1u7b3puTbv3nyAIxKRhigZi6ShPXvgnXe85REjGq/bJa8LAFt2b6HaVR/YwEQkKSVjkTS0dCns3An9+kGPHo3XzcrIojCnkGpXzbbyba0ToIjUoWQskoZeecW7P/HE5tXvmtcVUFe1SFCUjEXSUHk5dO4M45o5HU/NceMyJWORICgZi6Sha66BjRubNwMXQNf2Xst4U9mmAxiViDSkWdNhikjbk5np3ZpD3dQiwVLLWCSNlJfD/fdDWVnLHleTjNVNLRIIJWORNHLfffD978O557bscfFuarWMRYKhZCySJp5/HqZP95a///2WPVYtY5FgKRmLtHF79nhd0+ec4y3/+McwYULLnkOzcIkESwO4RNqwv/8drr8e3nzTW//P/4QZM1r+POqmFgmWkrFICK1e7Z2atGULbN1a9/4rX4ELLvDqbdzoJeJeveBXv4Kzz96314t3U+vKTSLBUDIW2U9Dh0JVFTjn3aqra2833wyXX+7Ve+IJuOqqutsTb9u3Q3a2V/fcc+Htt5O/3re+VZuMzzkHHn4YpkyBwsJ934e+HfsCsGb7mn1/EhHZZ0rGIvuppAQqK5Nv27GjdrmyEjY30gvsXO3ysGHQrh106eLNpJV4P3x4bb3cXLjiiv0KH/BaxnmZeXxZ/iXbK7ZTmLMfmV1EWkzJWGQ/LV1am0jbtat769y5tt4FF8BZZ+1dp107MKttFQM8+mjr7oOZ0a9TP1ZsWsFn2z5jWI9hrRuASMQpGYvsp8GDm1cvN9e7hdXBHQ9mxaYVfPrlp0rGIq1MpzaJCAAHFx4MwGfbPgs4EpHoUTIWEQD6deoHKBmLBEHJWEQAr5sa4NNtnwYciUj0KBmLCFCbjNUyFml9SsYiAkC/jl43tVrGIq1PyVhEAOhT2AfDWL9jPeWx8qDDEYkUJWMRASA7I5sh3YZQ7ap5/4v3gw5HJFKUjEWkxoheIwB4e0MDc3GKyAGhZCwiNUYeNBKA4vXFAUciEi1KxiJSY2zfsQC8tPolXOJk2SJyQCkZi0iN0X1G0zO/JyVbSnh59ctBhyMSGUrGIlIjs10mVxzjXQZqzrtzgg1GJEKUjEWkjouOugiABcsXsL1ie8DRiESDkrGI1DG462BO7ncyOyt38ovFvwg6HJFIaFYyNrMJZrbSzErM7Lok23PM7E/+9jfMrH/KIxWRVnPzKTcDcNu/bmP+svnEqmMBRySS3pq8nrGZZQAPAacDa4ElZrbIObcsodq3ga3OuUFmNhm4E7jgQAQsIgfeyf1P5vwjzmfesnl8fd7X6V3QmzFFY+jXsR8jDhpBYU4h+dn55GbmMrLXSNpntQ86ZJE2rclkDBwLlDjnVgGY2ZPA2UBiMj4buMlffgr4pZmZ07kRIm3WnHPmcEyvY5j9zmw+3PwhC5YvSFrvg2kfcET3I1o5OpH00pxk3AdYk7C+FvhKQ3WcczEz2wZ0BTYlVjKzqcBUf3Wnma3cl6BTpBv14osY7X909z+l+z70pqGpeqrWEuXPHrT/Qe5/v4Y2NCcZp4xzbhYwqzVfsyFmVuycGxV0HEHR/kd3/6O876D91/6Hc/+bM4BrHdA3Yb3IL0tax8wygY7A5lQEKCIiku6ak4yXAIPNbICZZQOTgUX16iwCLvWXvw78XceLRUREmqfJbmr/GPCVwHNABjDbOfeBmd0CFDvnFgG/Bf5gZiXAFryEHXah6C4PkPY/uqK876D91/6HkKkBKyIiEizNwCUiIhIwJWMREZGAtflkbGZXmdkKM/vAzH6eUH69Pz3nSjM7I6E86dSe/gC1N/zyP/mD1Rqd6rOh12htZna1mTkz6+avm5k94Mf2npmNTKh7qZl95N8uTSg/xsze9x/zgJmZX97FzF7w679gZp2beo1W3O+7/M/+PTNbaGadErZF5vNviaamtg0zM+trZi+Z2TL/9/7//PIWf0dT9TsIgpllmNnbZvaMv56y725Lfx+tzcw6mdlT/u9+uZkdlzafv3Ouzd6A/wO8COT46z38+yOAd4EcYADwMd7gswx/eSCQ7dc5wn/MXGCyv/ww8B1/eRrwsL88GfhTY68RwHvQF29w3adAN79sIvAXwIAxwBt+eRdglX/f2V/u7G97069r/mPP9Mt/DlznL18H3NnYa7Tyvo8HMv3lOxNii8zn38L3q8H9bws3oBcw0l8uAD70P4cWfUdT+TsI6H34IfA48Ewqv7v78vsIYN9/D/ynv5wNdEqXzz/wH9h+fjBzgXFJyq8Hrk9Yfw44zr89V7+e/8ZvovYPe029+GP95Uy/njX0GgG8B08BRwOrqU3GjwBTEuqsxPtDNgV4JKH8Eb+sF7AiobymXvyx/nIvYGVjrxHgd+E/gD9G7fNv4XuUdP+Djms/9udpvDnzW/QdTeXvIIB9LgL+BpwKPJPK7+6+/D5aed87Ap/gDzyu/7m29c+/rXdTHwqc6HefvGJmo/3yZFN49mmkvCvwpXMuVq+8znP52+NTfTb0XK3GzM4G1jnn3q23qaX738dfrl8O0NM597m/vAHo2cRrBOVbeP/JQkQ+/33QFmNOyu9yHQG8Qcu/o6n8HbS2+4BrgGp/PZXf3X35fbSmAUAp8Du/m/43ZpZPmnz+rTod5r4wsxeBg5JsugEv/i543QqjgblmNrAVwzvgmtj/H+N11bYK55wzs1Y9F66x/XfOPe3XuQGIAX9szdgkGGbWAZgPfN85t90/rAe0znc0iN8BgJl9FdjonHvLzE5p7dcPgUxgJHCVc+4NM7sfr8u4Rlv+/EOfjJ1z4xraZmbfARY4r+/gTTOrxpsEvLEpPJOVbwY6mVmm/99fYv34c621ulN9Nmea0P3W0P6b2ZF4/ym+6/8xKgL+bWbHNhLbOuCUeuUv++VFSeoDfGFmvZxzn5tZL2CjXx7o/seZ2WXAV4HT/O9BU7G1qc8/xdpizHWYWRZeIv6jcy5+GamWfkdT+TtoTccDk8xsIpALFAL3k9rvbkt/H61pLbDWOfeGv/4UXjJOj8+/tfv9U3wM4b+AW/zlQ/G6HgwYSt0BCqvwBidk+ssDqB2gMNR//DzqDlCY5i9/l7qDIOb6y0lfI8D3YjW1x4zPou7AhTf98i54x1w6+7dPgC7+tvoDFyb65XdRd+DCzxt7jVbe5wl4l/LsXq88cp9/M9+vBve/Ldz879qjwH31ylv0HU3l7yDA9+IUagdwpeS7uy+/jwD2+5/AYf7yTf7nkhaff+A/sP38YLKBx4ClwL+BUxO23YA3MnAl/og4v3wi3ijMj/G6OuPlA/0PosT/4sVHaOf66yX+9oFNvUZA78VqapOxAQ/5sb0PjEqo9y1/X0qAyxPKR/nv48fAL6mdna0r3oCRj/BGrndp6jVacZ9L8P4Be8e/PRzVz78F71nS/W8LN+AEwAHvJXzmE/flO5qq30GA78Up1CbjlH13W/r7CGC/hwPF/nfgz3jJNC0+f02HKSIiErC2PppaRESkzVMyFhERCZiSsYiISMCUjEVERAKmZCwiIhIwJWORA8C8q2g1dVvt151jZmubeMpWYWarzeyxFD/fnGbUmxN/P0SiKPQzcIm0UcfVW1+IN4nCTQllFa0WjYiEmpKxyAHgnHs9cd3MKoBN9cv3l5nlOOeU1EXaOHVTi4SEmY0ws3+aWZl/EfP/qrf9Mr97+yQzm2dmX+JdtQgzyzTvgvErzKzCzNab2T1mlpvw+Ewzm2FmH5tZuZltMrN/mdkJSWKZ7F+8fZeZFTdQ5yIzezfhuf7gz9vb1H6eZmb/9h/3sZldsS/vl0g6UctYJBwK8S4Yfx9wC3A58CszW+mce6le3T8CTwBfp/Y3/BjwNeBO4DXgcGAG0B84z69zLfADvKkQ3/FfcxTeXL2JTgQOA34ClPvP84yZ9XfOfQlgZlPxrgP7J7xr3vYGbgO+YmYjnXM7k+2kmR0OPIs3peFkvPmRbwI6AHsaf4tE0peSsUg4FOBNvv8SgJn9AzgD7wLn9ZPxU865a+IrZnYicAFwqXPuUb/4RTPbAjxmZsOdc+/gHcd+3jl3f8Jz/U+SWAqB4c65rf7zbwCW4M1b/LiZZeAl6Jedc5MT4liBN5H/t4AHGtjP6cAOYLxzbpf/uNfw5gJe39CbI5Lu1E0tEg5liS1g/zjwh8DBSeourLc+AagEnvK7ojP9S+Y9728/yb9fAkw0s5+Z2Qlmlt1ALIvjidj3vn8fj+UwoAf1rh/tnPsX8ClwckM7ifcPwbPxROw/bg3waiOPEUl7SsYi4bA1SVkF3pV36vu83noPvCuY7QKqEm7xa6529e9vA34KTMJrwW42s9+ZWbd6z7clcSVhgFg8lni3dv04ADawd7d3ol7AF0nKk5WJRIa6qUXanvqXWtuMd2z3xAbqrwdwzlXhHVO+08wOAr4K3Au0x+vmbq54sj4oybaDgLcaeeznQM8k5cnKRCJDLWORtu+veK3Wjs654iS3vY7FOuc2OOd+g3dt1mEtfL2VeC3ZyYmFZjYW6Ae83MhjF+N1lecnPK4vcHwLYxBJK2oZi7RxzrmXzewJvGPG9+JdBL4abyT1ROBa59yHZvY03sQj/8brFh+Bd7z5kRa+3h4zuxF4xJ+t6zGgD/AzvIuvz27k4bcC5wPPm9ldeN3rN6Fuaok4JWOR9HARcBXeSOYb8I43rwaeozbR/QMvEX4Xr2v6M+DneEm0RZxzs8ysDPgR8DSwE++UpWsSB2cledxyM5sI3IV3WtQ6vK7z44BTWhqHSLow5+offhIREZHWpGPGIiIiAVMyFhERCZiSsYiISMCUjEVERAKmZCwiIhIwJWMREZGAKRmLiIgETMlYREQkYP8fiU7Rm6dPddYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 使用matplotlib 绘制精度和召回相对于阀值的函数图\n",
    "def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):\n",
    "    plt.plot(thresholds, precisions[:-1], \"b--\", label=\"Precision\", linewidth=2)\n",
    "    plt.plot(thresholds, recalls[:-1], \"g-\", label=\"Recall\", linewidth=2)\n",
    "    plt.xlabel(\"Threshold\", fontsize=16)\n",
    "    plt.legend(loc=\"upper left\", fontsize=16)\n",
    "    plt.ylim([0, 1])\n",
    "\n",
    "plt.figure(figsize=(8, 4))\n",
    "plot_precision_recall_vs_threshold(precisions, recalls, thresholds)\n",
    "plt.xlim([-700000, 700000])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAF5CAYAAACV7fNGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAApsklEQVR4nO3deZQU5bnH8d8zM2yyOIMiCBLccCEatwEVRUeNMXrdokaNGr2iksQtXmOOKMRoRGPMFZcoejDgvqG5EmNiokQRFTEOcYn7Bi7EhU02Hdbn/vH2pHqGWXpmuqt6+X7O6VNvVVdXPRQcfl3Vb71l7i4AAFD8ypIuAAAAxIPQBwCgRBD6AACUCEIfAIASQegDAFAiCH0AAEoEoQ8AQImIPfTNrK+ZPdPC+53M7FEzm2lmI+OsDQCAYhZr6JtZlaQ7JHVvYbVzJNW6+3BJh5pZz1iKAwCgyMV9pr9W0nGSlrawTo2kKan2TEnVOa4JAICSUBHnztx9qSSZWUurdZc0L9VeKqlv4xXMbJSkUZJUUVG1W69eWzZ4f9kyafVqqUcPadtts1A4AAB5ZPbs2QvcvU9bPxdr6GdouaRukpZI6pGab8DdJ0qaKEnV1dVeW1vb4P2nnpL231+qrpZuvll68EHp73+X3ntPuvPO8B4AAIXKzD5sz+fyMfRnS9pb0kOSdpI0q70bmj5d2n77hsseekiaP18aMkTaccf2F+kuffKJVFkp9aTXAQCgACQa+ma2v6Qh7n5j2uI7JP3FzEZIGiLphbZutyytp0L37tJxx0mvvCLNnh3O/G++WdpuO+nNN9f/7Ny54UrBwIHSt78dwn3ePOnFF8M23n47en31VfjM/fdL3/++tHat1KlTtC13acECac6c8PryS+nYY6Wqqrb+iQAA6DjLx0frmll/hbP9v7n7kpbWbery/ldfSaNGhWA/91ypV68Q9Gee2fCzzz0n7b679PTT0pQp0uOPh3Cud+CB0muvSZ9+mnntRx0Vwv+DD8K2lq/344R09NFSly7SEUdI69ZJH38s7buvNGxY5vsBAJQuM5vt7m3u6J6Xod8WTYV+U1avDqG+zTbhVW/TTRuGemVlOCNPV1kpDR0q7bpr+CKx3Xahg+DMmdKhh7a83169pC22kN5/v+kvAPW23FJ6+OHwZWHu3NDv4FvfavWPBQAoQYR+GxxyiPTYY9H8lltKxx8vHXlkCPZ775WeeUbae29pjz2krbdu+JNBusWLpXfeCXcMPPigtNlm4QvBVluFsK+qkupvVnjhBem226TPP5emTpV22y2s/8c/Nr3tnXcO27r33rB/d+mzz6S6OmnzzaWFC8PVhGXLpG9+M/y00Lt3uNJQXt6mQwIAKCCEfhvMmiX9139JI0ZI558fpi3fRZg77uHS/gsvhC8Jm20W7jRIV1EROh6+/760YkXm237pJemjj9Z/vflmuJrxzW9KffpI06aFLwnuYXlZWbhCsWhRuArS+PXvf4c+DjvsEL4wXXedtHKl1LVr+Lli5UqpW7fm/7zu0ZeY1o57XZ20dGmos35d97Bs0SKpX7/m9wUAxYrQL3Dr1kVXE+bMkWbMkP77v1v+TK9eIfyyZYMNos6J7VVWFv4s9Xr3DuE8ZEj4wvLFF9LXX4f3KiqkNWtCe9iwcOXi/ffDrZYLFoRX+k8i228f1lm0KPpcYyNGhKsjc+aEzpVHHRXGa+jUKXyuZ89wJaSuLvxss3at1LlzmOfLA4BCQegXoS++kO6+O/Q72Hrr8KqsDIFWWRn9dLB6dQjbt98OyzbbLHx+yBDpG99o+Bo4MJzp9+olnXBCy/vv1Svsu3//MK1vV1RI774r3Xhjy5/Ppe7d23bVI1Ndu4Zje8AB4UvCyy9LF10UOl/275/9/QFAexD6aJevvgqdB8vKwpeFnj3D5fuqqnDm35r588N6S5eGs+9NNgln46tXh3EMysvDTwKbbhre69Mn9Gno0iUE7CuvhJ8U+vULZ9t1dWGdjTcOr169wpeU+fOljTYKr969w+elcGVg7VrpiSekZ58NnSynTJHeeKNhnWbhZ4GOGj063Ha5cGG4YtGtW7hK0atXx7cNAJki9IFmLF8evpiUlUmrVoXL+atXhy8ky5aF+VWrpEmTwpeXnj3DF4877wxjNqT/XNGS8nLp0kvDF5KRI8MXFADIBUIfyJElS6Tx46Ubbgh9AzbdVHr1VWnAgDBwU2vS+znssot02mmhE+WAAeFqR//+3G0BoG0IfSAhn3wiTZ4cbsP84IPwJaG9zKSf/lQaNy70WwCAphD6QB6ZPz+MtLhoUehL8N570tix4RbHTP+5jhgRRoQcPTqM6bByZfgisGRJ6OTZvXvorLnRRlLfvg2HgAZQ3Ah9oADV1YX+BX/6UxifYfLk7Gz3/ffDFwwAxam9od/MOHMA4tC1a+g4eMIJoSOhe7j18s47pQsuCHdRVFSEM3kp8+czbLVVuJNh8uTs3LUAoDhwpg8UOPfw0KhXX5V+85twy2VTKirCGASPPCLtuWesJQLIMs70gRJlJtXUhCdKzpsXXk09rGnNmjDK4fDh4TMDBoRnTLz8cugvsGZN5rcnAihMhD5QZPr3D4MeuYeBi/7+9/CFYLvtGq73739L++wTbiPs2jV0BCwvD/M9eoT36+qS+TMAyA1CHyhiZWXhMc3XXx9GNly3LtxW2NLAQS+/HIY4HjAgjDhoFl4bbhgeDnXPPaHzIYDCw2/6ALR2bRiS+fnnpUMPzSzUu3ULDzXaZJPwpeDcc6PnPgDILW7ZA5B1a9aEKwRffBHuLrjvvsw+V14exibYfPOclgeULEIfQCyWL5cuv1x6/fUwFkB9Z8Dm3HFHGERoyJDQb8As3EkAoP0IfQCJcQ9PU6yokK66Srr22pbXr6iQTjpJGjMmjCkghS8DADLDLXsAEmMW7hrYZJPwcCJ36aKLwiOTm7JmjXT77dLgwaGzYVlZ6B9w001hmOIVK8KXCDoMAtnFmT6AWKxcGR5j/Npr0sknt/yTQDozaY89wgiFQ4dKAwfmtEygIHCmDyCvdekSAnzHHaWXXgpXA1avlj76SDr11Ia/83fpErXdw10FRx8tfeMbYRunnspVAKA9CH0AiamoCGfukyeHEHcPYwnU1YWfAJ55RjrggPCUwXS33x6uGlRWSscdJ33+eRLVA4WH0AeQV+o79JWXS3vvLU2bFn7nd5emT2+47pIl0pQpoe/AlVeGpwuuXRt7yUDBIPQBFIx99w3hv3hx6PRXUxO9N2aMtPXW4erB976XWIlAXiP0ARScykrpzDOlp56S/vWv9d+fOjVcMSgrk5Yujbs6IH8R+gAK2g47hLP/+gcMpXMPzww46aTQBkodoQ+gaJSVhXCvqwujBda7557w3s9+llxtQD4g9AEUnS5dQqe+JUsaLh8/Plz2v/VWHhuM0kToAyhavXqFM/8ZMxouHzUqPCWwa1dp4sQwAiBQCgh9AEVvxIgQ/qec0nD5ypXSj34k9egRrgDcfHN4xDBQrAh9ACXj9tujDn/XXLP++2eeKXXvHr4AbLNNuPXvpZfCgEFAMSD0AZScsjLp/POjXv8PPLD+Ou++G27923XXMFAQUAwIfQAl79hjoy8AX30VLvk3Zib99KfSzJnc/ofCRegDQJpu3aRbbom+BAweHL13ww3SXnuFKwVDhkjz5iVXJ9AehD4AtODtt6XzzpMOOqjh8jfflDbbLFwBeOutREoD2ozQB4AWmEnXXiv99a/hzP8f/5A237zhOttvL/3mN4mUB7QJoQ8AbTB0qDRnTvgCcP310fLRo6UttpDmzk2sNKBVhD4AtNO550oLFkTzc+eG4D/9dGn58sTKAppF6ANAB2y0Ubjv/6qromWTJkk9e0pjx0pr1iRXG9AYoQ8AHVRWJl14ofTyy1LnztHyK66QOnWSLrsssdKABgh9AMiSnXYKQ/u+/XbD5ZdeGp4DsHp1ImUB/0HoA0CWbbNN6Oj32WfRsmXLGl4FAJJA6ANAjvTtG87uN944WmbGY32RHEIfAHKookKaP7/hsm7dQvi/914yNaF0EfoAEIOmntQ3eHAI/xdfjL8elCZCHwBiYBaN53/ZZQ2f3DdsWHj/rLO4xQ+5RegDQMwuuSSE+y23NFw+YUK4xY+BfZArhD4AJORHPwpn/rNmNVzes6f04YfJ1ITiRugDQMJ23z2E/0knRcsaP9QHyAZCHwDyxF13SddcE82b8fQ+ZBehDwB55PzzpUGDovnRo6NOgEBHEfoAkGfmzpVmzmy4rIz/rZEF/DMCgDy0557h7L5372iZmfT558nVhMJH6ANAHlu4sOF8v37S8OHJ1ILCR+gDQJ5zl447Lpp//nnG8Ef7xB76ZjbJzGaa2dhm3q8ys7+Y2TNmdktT6wBAqbn/fmnFiobLunULj/IFMhVr6JvZUZLK3X24pP5mNriJ1X4o6W53HyGpp5lVx1kjAOSrDTYIZ/1HHBEt69o1nPVff31ydaFwxH2mXyNpSqr9pKS9m1hnoaRtzaxS0kBJH8VSGQAUiKlTpd//vuGy886TPvkkiWpQSOIO/e6S5qXaSyX1bWKdZyUNlnSupLckLW68gpmNMrNaM6ud3/iZlQBQAk47LTy576mnomUDB/K4XrQs7tBfLqlbqt2jmf1fKenH7v4rhdA/tfEK7j7R3avdvbpPnz45KxYA8pmZVFMjXXtttGxwUz+aAilxh/5sRZf0d5I0t4l1NpC0o5mVS9pdEuNQAUALzjsvDOFbb+ONEysFeS7u0J8q6YdmNl7SsZJeN7Nxjdb5taSJkpZI6i3pvlgrBIAClP6wnoULpTFjkqsF+cs85gGdzaxK0oGSZrj7Zx3dXnV1tdfW1na8MAAocHV14Ta+dI88Ih12WDL1IHfMbLa7t/nuttjv03f3xe4+JRuBDwCIdO0qffppw2WHHy5tu20y9SD/MCIfABSRfv3CvfwPPRQte+cdafvtuaUPhD4AFKWjj5ZWrYrm33or3NKH0kboA0CR6tQpnPX/7nfRsgMPTK4eJI/QB4Aid/bZ0gEHhPa0adI//5lsPUgOoQ8AJeDBB6P2bruFy/0oPYQ+AJSAqipp9uxofvvtpVdfTa4eJIPQB4ASseuuDcfq32knaa+9pM8/T64mxIvQB4ASUlPT8Ha+mTPDbX4oDYQ+AJSYo4+WFi+Wxo6NlplJjz+eXE2IB6EPACWoslK6/PKGD+c56CDppZcSKwkxIPQBoITNny/deGM0v+uu0n77JVcPcovQB4ASd9ZZ4f79etOnh8v9KD6EPgBABxwgLVjQcNnBBydTC3KH0AcASJI22khaty6a/+tfpXvvTa4eZB+hDwD4DzNp0aJo/sQTpT/9Kbl6kF2EPgCggaoqae7caP7ww6XbbkusHGQRoQ8AWM+gQdKLL0bzI0dKzz2XXD3IDkIfANCk6mrpk0+i+b33lj76KLl60HGEPgCgWQMGhFv46g0alFgpyAJCHwDQon33lSZMiObTL/ujsBD6AIBW/eQnUXvYsOTqQMcQ+gCAjKTfs//HPyZXB9qP0AcAZOQHP4jaRx6ZWBnoAEIfAJCxhx+O2ttsk1wdaB9CHwCQsSOOiNrvvsugPYWG0AcAZMxMWrUqmh85UjrttOTqQdsQ+gCANunUSZo9O5qfPDmEP/IfoQ8AaLNdd5VWrozmb7tN6ttXck+uJrSO0AcAtEvnzg2H5f3iC2no0OTqQesIfQBAuw0c2PCMf/ZsaautkqsHLSP0AQAd0rmztG5dNP/BB9KhhyZXD5pH6AMAOsxMqquL5v/85zBcL7/x5xdCHwCQFV26hDP+ysow/+KLUhkpk1f46wAAZI2ZtGiRVFUVLdtww+TqQUOEPgAgq8xCT/56S5dKZ5+dXD2IEPoAgKyrqGjYue+mm5KrBRFCHwCQE2bSrFnR/COPJFcLAkIfAJAzw4ZF7fSH9SAZhD4AIGfMpOnTo3kezpMsQh8AkFP77itttFFoT54sPftssvWUMkIfAJBzH34YtUeMSK6OUkfoAwByrnt3aerUaP6wwxIrpaQR+gCAWKR35Hv0UWncuORqKVWEPgAgNsuXR+1f/EKaOTO5WkoRoQ8AiE337tLixdH8XntJ772XXD2lhtAHAMSqslJ64IFofvBg6fLLEyunpBD6AIDYHXtsuH2v3iWXhHv6kVuEPgAgEaeeKi1Y0HAZwZ9bhD4AIDEbbSQtXNhwGcGfO4Q+ACBRvXtL7tLWW0fLHn88uXqKGaEPAMgL77wTtQ86aP0rAOg4Qh8AkBfMpNraaH7AgORqKVaEPgAgb+y2m3TRRaG9cqV00knJ1lNsCH0AQF755S+j9j33SE8/nVwtxYbQBwDklS5dpM8/j+ZraqQ1axIrp6gQ+gCAvLPJJtI//xnNd+oUevijY2IPfTObZGYzzWxsK+tNMDMevggAJWqXXaQrrojmv/Od5GopFrGGvpkdJanc3YdL6m9mg5tZb4Skfu7+pzjrAwDkl4svjtrTpq0/gh/aJu4z/RpJU1LtJyXt3XgFM+sk6VZJc83siMbvAwBKy7JlUbtPHy7zd0Tcod9d0rxUe6mkvk2sc7KkNyRdLWmYmZ3TeAUzG2VmtWZWO3/+/JwVCwBIXo8e0rhx0fz3v59cLYUu7tBfLqlbqt2jmf3vImmiu38m6W5J+zVewd0nunu1u1f36dMnZ8UCAPLDmDHRYD1/+IO0YkWy9RSquEN/tqJL+jtJmtvEOu9J2jLVrpb0Ye7LAgDku/TR+nr0SK6OQhZ36E+V9EMzGy/pWEmvm9m4RutMkrSfmc2QdKak/423RABAPurXT7rggmj+wQeTq6VQmcfcI8LMqiQdKGlG6hJ+h1RXV3tt+tc/AEDRWrtWqqiI5hctkqqqkqsnKWY2292r2/q52O/Td/fF7j4lG4EPACgt5eXSnDnRfO/e4YsAMsOIfACAgrL55tLEidF8+pk/WkboAwAKzhlnNByh79Zbk6ulkBD6AICC9Le/Re1Ro5Kro5AQ+gCAgnX77VH7nXcSK6NgEPoAgIJ14olRe9ttk6ujUBD6AICCVVEh3XhjNH/mmcnVUggIfQBAQTvrLKlXr9C++WYeyNMSQh8AUPA+TBuwvYxkaxaHBgBQ8CorpaFDo/lXXkmslLxG6AMAisI//hG1d945sTLyGqEPACga110XtT/6KLEy8hahDwAoGuecE7UHDUqujnxF6AMAikZZWcMBeyZNSqyUvJT10DezbtneJgAAmTrllKh9+unSunXJ1ZJvWg19M+tsZiNS7XIzO6yVj4wxs8uzUh0AAO3wr39F7ZEjk6sj32Rypt9b0rRUu0LS/a2sv4mkDTtSFAAAHbHDDtIGG4T2HXckW0s+yST0V6ZecveVktakv2lm/2dmfdIWbSnp1axVCABAO8ycGbVPOim5OvJJJqG/TtJaM7vVzJZJ6mFmi81smZkdKOlISS+bWf2wCLtIej435QIAkJmddora99yTXB35pC0d+a6XdISkFUoFferzSySNkfRXM/u5pK/d/fXslgkAQNu98ELSFeSXigzW2VeSu/trkmRma9z9aTNbkHrf3f321PxUSeNzUyoAAG3zzW9G7ddfbzhfilo80zezhxWCPBPDUtMuHSkIAIBs6d49an/ve8nVkS9au7x/vaThkmRme5rZqZI6m9nJkgam1qkws99LOlbSAZKOMTPLVcEAALTFBReE6bvvJltHPmgx9N19uqQ3JZmkGknnKZzJ/0zSBpLqJPVIvT/M3Z+WNE/SPrkqGACAthgzJmpffXVydeQDc/eWVzDbUNIcd+9tZmWSFrv7hqn3Okta5O490ta/OrXsqhzW/R/V1dVeW1sbx64AAAUq/frzunUN5wuRmc129+q2fq6tw/B2lZQ+zK5JeqDROq9J2q2thQAAkCvz5kXtshJ+6kymf/QuZna2pJGSzjOz083sBIXf+682sx5p674v6cEs1wkAQLv17y8df3w0X6q9+DO5ZW+tpLcknSip/reACkndU69+kjqZ2TuSHpc02d2n5KBWAADa7d57pWeeCWf9b7wh3XSTdNZZSVcVr1Z/089oI2bfkLS/pOMlHShpH3d/rsMbzgC/6QMAMrVqldQl7cbyLERgIuL6TV9m1sfMNk5f5u4fufvt7v5dSTvFFfgAALRF587SxInR/G9/m1wtScjk0bpdzexnFnSVdIakk5tbv37kPgAA8tHpp0fta65Jro4kZHKmv1bS/0jaQdIEhXvzV5nZbDP7xMw+aPR628wuzWHNAAC0m5l0442h/fnn0kcfJVtPnFoNfXdfLWm1QtjXKTxad7WkKoXOfd0knZo2fU3SBWZWnqOaAQDokFGjovagQcnVEbfWxt7/rpnVKIzCVy2pr6St699PjcD3dWpal5peI+lwhUfyAgCQdzp1km64IZq/+OLkaolTi733zWy+wtl9P0mfSeqlcJY/RtKF7r6lmX2Qms5x9y3iKDodvfcBAO3hLvXsKa1YEXr019UlXVHmctJ73937uPtASZ8ojL1/p6TLm1u9rTsHACApZtKsWaG9cmUYnrfYtTo4T+q3+fr1OqemZeEtu0RSVaOpJMndf5XtYgEAyKYddojagwZJH3+cXC1xyGREvq6p11eS/iGpd2r5Awq/8d+l0KnvTkkbSypX6AMAAEDB+OQT6csvpcrKpCvJnVZD391XmNlZkla7+yQzO0bhqXuzzew0SVu5e4l0gQAAFJtVq8KgPZI0fHgYordYZToi3w8kvWdmxyuc4b9hZrdLGi1pWo5qAwAg5zp1ko48MrTffLNwh+bNRItn+qmQ/1rS7ZL+JWm71FsHSfpSoVNfDzM7PO1j9Zf3H3T3tVmuFwCArJs0SZo6NbSfflqqqUmymtxp7fL+LyWtVOiZ75Is9fq/1Pz7kpanlqVvs4ukR1PvAQCQ13r3Dr353aX99ives/3Wbtnb3t13lrSPpOclXagQ9t+T9BdJlZLuk1Tt7rukXju6+zbuTuADAArGo49G7Z/8JLk6cinT3/SnKPTUf0PhrP4xdz9M0iGSjpP0nJlZC58HACCvHXJI1L7lFmn16uRqyZVMQ/+/3f0Yd/9E0hbuvkqS3L1W0h6SzveWhvYDAKAALF4ctR97LLk6ciWj0Hf3z9PaHzZ6b427z8x2YQAAxK2yUqpI9XarfxJfMcn0TB8AgJLw4x+H6RNPJFtHLhD6AACkGTkyan/6aXJ15AKhDwBAmp13jtq7755YGTlB6AMAkMZMOuOM0P744zBMb7Eg9AEAaOS666L2PfckVkbWEfoAADSywQbS5puHdvpv/IWO0AcAoAlTpkTt9IF7ChmhDwBAE4YOjdqPPVYc4/ET+gAANGPJkqg9dmxydWQLoQ8AQDN69ZI23TS0r7wy2VqygdAHAKAFDz0UtZctS66ObIg99M1skpnNNLMWL5SYWV8zeymuugAAaMrw4VH7oIOSqyMbYg19MztKUrm7D5fU38wGt7D6/0rqFk9lAAA0rz74n38+2To6Ku4z/RpJ9TdBPClp76ZWMrP9Ja2Q9Fk8ZQEA0LzJk6P2tGnJ1dFRcYd+d0nzUu2lkvo2XsHMOku6RNLo5jZiZqPMrNbMaufPn5+TQgEAqLfttlH7rruSq6Oj4g795You2fdoZv+jJd3k7l82txF3n+ju1e5e3adPn+xXCQBAI+ecE6Z33plsHR0Rd+jPVnRJfydJc5tY59uSzjKz6ZJ2NrPfx1MaAADNO/PMqP3OO8nV0RFxh/5UST80s/GSjpX0upmNS1/B3fdx9xp3r5H0srufHnONAACsZ7vtovYvfpFcHR0Ra+i7+1KFznyzJO3n7q+4e7O37qWCHwCAvHDMMWE6ZYq0enWytbRH7Pfpu/tid5/i7vTMBwAUlPRH7t53X2JltBsj8gEAkKEBA6Ke/Keckmwt7UHoAwDQBoceGrXr6pKroz0IfQAA2uCqq6J2twIbN5bQBwCgDSoqpNFpw8fdemtytbQVoQ8AQBv9+tdR+7nnkqujrQh9AADaYcKEML3jDmnt2mRryRShDwBAOxxySNTeY4/k6mgLQh8AgHYYNEj66U9Du7ZW+uc/k60nE4Q+AADtdO21UfuAA5KrI1OEPgAA7WQm/fKXof3ll4mWkhFCHwCADjj77Kg9Y0ZydWSC0AcAoAM23jhq77tvcnVkgtAHAKCDLrssai9cmFwdrSH0AQDooDFjovakScnV0RpCHwCADiovl048MbQvvDDZWlpC6AMAkAW/+EXUnj8/uTpaQugDAJAF224bte++O7k6WkLoAwCQJQceGKbnn59sHc0h9AEAyJIbbojaDz+cXB3NIfQBAMiS7baL2rNmJVdHcwh9AACyaNy4MH3yyWTraAqhDwBAFnXpEqa1tcnW0RRCHwCALDr99Ki9YEFydTSF0AcAIIsqK6P2HXckVkaTCH0AALJsv/3CNN9+1yf0AQDIsqFDw/SDD5KtozFCHwCALKupCdO33kq0jPUQ+gAAZNnuu0ftfLpfn9AHACDLeveO2meckVwdjRH6AADkwM9/HqZffploGQ0Q+gAA5MAJJ4RpPj1ml9AHACAH+vcP05Urpa++SraWeoQ+AAA5sMkmUfuyy5KrIx2hDwBAjnzrW2F6443J1lGP0AcAIEd+9asw/eoryT3ZWiRCHwCAnDn44Kj95pvJ1VGP0AcAIEc6d47aEyYkV0c9Qh8AgBw65pgwvemm5HvxE/oAAOTQFVdE7bFjk6tDIvQBAMipbbaRDjsstK+9NtlaCH0AAHLs4ouj9rnnJlcHoQ8AQI7tsYfUo0do/+53ydVB6AMAEIOPP47aCxYkUwOhDwBADCoro/a//51MDYQ+AAAxqX8Iz7Rpyeyf0AcAICZ77RWmP/tZMvsn9AEAiMnxxye7f0IfAICYHHlk1J4xI/79E/oAAMSkLC1177svgf3Hv0sAAErXyJFhesst8e+b0AcAIEaHHBK16+ri3TehDwBAjI4+OmrH/QAeQh8AgJjtumuYXnNNvPsl9AEAiNlllyWzX0IfAICY1dRE7S+/jG+/hD4AADHr0UPq0iW0X3stvv0S+gAAJGCDDcL0oYfi22fsoW9mk8xsppk12WfRzDY0s8fM7Akze9jMOsddIwAAuXbQQWH6/PPx7TPW0DezoySVu/twSf3NbHATq50oaby7HyjpM0nfjbNGAADisNtuYbpiRXz7rIhvV5KkGklTUu0nJe0t6d30Fdx9QtpsH0lfxFIZAAAx2n33MH399fj2Gffl/e6S5qXaSyX1bW5FM9tTUpW7z2rivVFmVmtmtfPnz89NpQAA5FB1ddS+66549hl36C+X1C3V7tHc/s2st6TfSRrZ1PvuPtHdq929uk+fPjkpFACAXOrWLWqffLLknvt9xh36sxUu6UvSTpLmNl4h1XFviqSL3P3D+EoDACBe6U/amzkz9/uLO/SnSvqhmY2XdKyk181sXKN1TpO0m6QxZjbdzI6LuUYAAGJx/PFRO45e/LGGvrsvVejMN0vSfu7+iruPbbTOze5e5e41qdcDcdYIAECcDj44TOO4Xz/2+/TdfbG7T3H3z+LeNwAA+WbffcP0hRdyvy9G5AMAIEEHHBDfvgh9AAASVP+YXSn3nfkIfQAAElSWlsTjx+d4X7ndPAAAaM3Pfx6mf/hDbvdD6AMAkLDDDovaf/5z7vZD6AMAkLARI6LL/Icemrv9EPoAAOSB666L2qtX52YfhD4AAHngrLOi9ne+k5t9EPoAAOSBsjKpd+/Qnj49N2f7hD4AAHnigw+idi6G5SX0AQDIExtuGDr1SdKbb2Z/+4Q+AAB5ZI89wvTyy7O/bUIfAIA8MmRI7rZN6AMAkEeOPz5qP/VUdrdN6AMAkEe6dpUGDAjtadOyu21CHwCAPFM/LO+VV2Z3u4Q+AAB55sILo/aiRdnbLqEPAECe2XxzqXv30L7qquxtl9AHACAPVVWF6W9/m71tEvoAAOShBx6I2r/+dXa2SegDAJCHhg+XNt44tC+9NDvbJPQBAMhTkyeH6apV2dkeoQ8AQJ6qH4dfkpYv7/j2CH0AAPJUZWXU/tGPOr49Qh8AgDy2zz5heu+90urVHdsWoQ8AQB6r/11fkjp37ti2CH0AAPLYVltJAwdG8z/4Qfu3RegDAJDn5syJ2vff3/7tEPoAAOS58nKprq7j2yH0AQAoAF26SOPGdWwbhD4AAAXi4oulGTPa/3lCHwCAAmHWcMCetiL0AQAoEYQ+AAAlgtAHAKBEEPoAAJQIQh8AgBJB6AMAUCIIfQAASgShDwBAiSD0AQAoEYQ+AAAlgtAHAKBEEPoAAJQIQh8AgBJB6AMAUCIIfQAASgShDwBAiSD0AQAoEYQ+AAAlgtAHAKBEEPoAAJQIQh8AgBJB6AMAUCIIfQAASgShDwBAiYg99M1skpnNNLOxHVkHAAC0Tayhb2ZHSSp39+GS+pvZ4PasAwAA2i7uM/0aSVNS7Scl7d3OdQAAQBtVxLy/7pLmpdpLJW3dnnXMbJSkUanZlWb2WpbrxPo2lrQg6SKKHMc49zjGuccxjse27flQ3KG/XFK3VLuHmr7S0Oo67j5R0kRJMrNad6/OfqlIx3HOPY5x7nGMc49jHA8zq23P5+K+vD9b0eX6nSTNbec6AACgjeI+058q6Rkz6y/pYEnHm9k4dx/bwjp7xFwjAABFKdYzfXdfqtBRb5ak/dz9lUaB39Q6S1rZ7MQclIr1cZxzj2Ocexzj3OMYx6Ndx9ncPduFAACAPMSIfAAAlIiCCX1G8su91o6fmW1oZo+Z2RNm9rCZdY67xmKQ6b9TM+trZi/FVVcxacMxnmBmh8VVVzHJ4P+LKjP7i5k9Y2a3xF1fsUj9P/BMC+93MrNHU38XI1vbXkGEPiP55V6Gx+9ESePd/UBJn0n6bpw1FoM2/jv9X0W3ryJDmR5jMxshqZ+7/ynWAotAhsf4h5LudvcRknqaGbfxtZGZVUm6Q2H8muacI6k29XdxqJn1bGmbBRH6YiS/ONSolePn7hPc/YnUbB9JX8RTWlGpUQb/Ts1sf0krFL5coW1q1MoxNrNOkm6VNNfMjoivtKJRo9b/HS+UtK2ZVUoaKOmjWCorLmslHacwUF1zahT9XcyU1OKXq0IJ/caj9PVt5zpoXsbHz8z2lFTl7rPiKKzItHqcUz+bXCJpdIx1FZNM/i2fLOkNSVdLGmZm58RUW7HI5Bg/K2mwpHMlvSVpcTylFQ93X5rBHWxtyr5CCf2sjOSHFmV0/Myst6TfSWr1tyM0KZPjPFrSTe7+ZVxFFZlMjvEukia6+2eS7pa0X0y1FYtMjvGVkn7s7r9SCP1TY6qt1LQp+wolGBnJL/daPX6pM9Apki5y9w/jK62oZPLv9NuSzjKz6ZJ2NrPfx1Na0cjkGL8nactUu1oS/57bJpNjvIGkHc2sXNLukrg/PDfalH0FcZ++mfWS9Iykvys1kp+k76cP7NPEOntkcFkEKRke458ofHt/JbXoZnd/IO5aC1kmx7nR+tPdvSa+Cgtfhv+We0qarHAptJOkY9x9XhObQxMyPMbDJN0maZCk5yV9z92XJ1Buwav/fyDV12eIu9+Y9t4gSX+RNE3ScIXsW9vstgoh9KX/9GI8UNKM1CW5dq2D5nH84sFxzj2Oce5xjPNHatj6vSX9rbWT3YIJfQAA0DGF8ps+AADoIEIfAIASQegDkJl1T/WyBlDECH0AUrjXd42ZeQavyfUfMrMRGX4m/TUwwT8nUNIqki4AQF74hqSVklal5t+TNF7ShEbrTZf0adr8ytS0KsN9vCJpdburBNAhhD4AufvH9W0zGyppI0mPNB4V0Mw2VcMx1NemPt9gvaakxmCXpDUdqxZAe3F5H0Bjl0h61t1fS19oZl0VHrT0ftritY3Wmd/E5fwXG22f0AcSQugDkPSfznx3Stpf0plpy3unRgK7TGEo1Vdb2MzXkvZzd3N3k/Q/qWUA8gCX94ESZ2abSTpWIaDXSTrI3f+VtspaSX9T6Ox3hbu39EjldRkuA5AAQh8oYWbWRdLjClf9rpb0e3dvcGbu7kvMrJ+7L8xkkxkuA5AALu8DJczdVyo8oGM7SYdL+qqp2+wkLUibP7GFTXaV9FTa565NLQOQBzjTB0qcuy9NNb+W9AdJF7Sw+qtq+Tf6bbT+mT0d94A8QegDqLdO0nJ3n9vcCma2Ti38Rs/jrIH8xuV9AB3RnhMHhvsFEsKZPoB0p5jZKa2sk/7/RidJSv1+n6lOba4KQFZwpg+gnku6W2FI3eZeS9WwY16FpCX19+W39JK0RdpnACTA3NvyBR0AImZWIal7Jr/lm1mZpF4KXxL4jwdIAKEPAECJ4PI+AAAlgtAHAKBEEPoAAJQIQh8AgBJB6AMAUCIIfQAASsT/A70KpIZIAw2XAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "def plot_precision_vs_recall(precisions, recalls):\n",
    "    plt.plot(recalls, precisions, \"b-\", linewidth=2)\n",
    "    plt.xlabel(\"召回\", fontsize=16)\n",
    "    plt.ylabel(\"精度\", fontsize=16)\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "plot_precision_vs_recall(precisions, recalls)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "通过选择阈值来实现最佳的精度/召回率权衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred_90 = (y_scores > 30000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.990909090909091"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_train_5, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.020106991330012914"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "7、ROC曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAF6CAYAAAATeYHoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABJZUlEQVR4nO3dd3xUVf7/8deZSS+QAKEKgoo0ITRBEQXptl1dCyBFUBTd1f2Krh2QRXR19YuufVFA0K9Y9reLrgURUSwoVWNDRUFEek1vM3N+f8wkxhhIAsnc5M77+XjETGbu3PnkGuY959xzzjXWWkRERMT9PE4XICIiIuGh0BcREYkQCn0REZEIodAXERGJEAp9ERGRCKHQFxERiRAKfRERkQgR9tA3xjQzxnxwmMejjTGvGWNWGmMuD2dtIiIibhbW0DfGpAILgMTDbHYdsNZa2w841xiTHJbiREREXC7cLX0/MBLIOsw2A4GXQrdXAr1ruSYREZGIEBXOF7PWZgEYYw63WSKwLXQ7C2hWfgNjzFXAVQCJiYm9OnbsWLOFiohIjSlZ7d2W/2/ofn/AYku3s/gCttzzLFgo8gfwGlPytNA+LAW+AMW+AF6PIbfIDxYC1uLxmNB+ftlnOJVNurK5F7CWaK+HYn+ApNgoDMHfLS7ai6fMdqbMTvzFxRzc9TPFhfnEJadQkH1wr7U2rbo1hTX0qygHiAcygaTQz79irZ0DzAHo3bu3Xbt2bVgLFBEJN2stARsMSH/A4reWA7lF+AKWQp+fHZkFFPkCWGvxB4LBsiurgK3784jyevCYYIgGQoEIwe8Ba8kr8vP97hxapyaU3leyrQ39vCurkIN5RaQ1iINQLZbgdrbMPktub9ydQ3JsFNmFvho9Dv4K7osNfUEwPKrCGPAYg9cYjAnW3DQ5jobx0WQXFtOiQTzHNk4gv9jPSa0a4glt7zEGr8dQ7A/QqUUDjk9LCj7mCe7LYwzRUYaEmJqL19zcXI477jjiY6N5dsF8Ro4ciTFmy5Hsqy6G/jqgP/AvIB34xNlyRER+UVDsp7A4QGZ+MXnFPvblFFHsD+APBFuTP+3LIyHWGwpDWyY4fwnGr3dkcTCvONjKM6EgDtjSwD2QV8zP+/No3jCOTXtzOZhXHJbf7fOfMyvdZntmQZX3Vzbwo73BsDSY0gA1obDMyi8mYOGEpklEeYLBuXF3Nqcc17g0ZD3GYK1lV3YBJ7VsiMdjfhXE+3OLaNs4gRObJ+MxhuYN40hLisXjMaX7jI32kBQTPOaV9DjXCcXFxURHR5OYmMgjjzzCySefTLt27Y5qn46GvjFmENDZWvtombsXAG8YY04HOgOrHClORBxjrWV/bhH7c4soKA5g+SU0bbkQ3ZVVQGyUp7R1W7LN3pxCCn0Bvt+dQ0pCNIFQKK//6SCtU+PxBSwrf9jLCU2TSlurNvTatkwrNmBtaVf0pr25YT0OFQVslCcYgl6PIa/IT8P4aJokxZAQE0XT5FiivMGA84SCbldmAU0bxNKxeTKxUd7SwPslMAFj8PmD3eNNkmIJ3lVmGw8YDEX+AI0SY4j2ekLbBO8P7jN4u2RbYyDG6yE1MYbk2KjSrnapmi+++IJRo0YxY8YMLr74Yi655JIa2a8joW+tHRj6vhxYXu6xLcaYoQRb+9OttRX15ohIHRQIWIr8AQ7mFbPtYB4+f7Arujhg+WlfLlv25bE/r4ivtmXRrGEcPn+AlT/sAyAu2kMgAL5AgNo+/bphxy9jib/cdrhxxYfWIC6KrAIfJ7VqwA+7czn1+MbBMDaGLfvz6NEmBU9JEIaCtqRL2QBZBcU0TY7j+KaJv+o2LtnW57ekJESTFBtFtNdD2yYJNdplLHWTtZYnn3ySG264gZSUFBo1alSj+6+Tf0HW2u38MoJfRGpJQbGfgA0GcyBA8DxxXlEwqP0BPv85k4JiPxt355CZV1w6OGrVpn20bZzI2i0HaNkwjn25RRT6AtV67W93ZZer5bfPbxgfTWZ+MV1D51Qp10I1GALWsv1gPumtU0pbtyXb7Mku5Pi0RPKK/HRp2QCv10OUx5Bb6KNt40SivIYoj4eG8dG/aq3+quUauk3o/hYN4xS+Uiv279/PlVdeyb///W9GjBjBggULaNq0aY2+hv5yReogf8CSlV+ML2DxBQLsyykqDedCX4Dd2YVEeQw7MwvwB2zpQCR/ALbsyyUmKtj9unbLAY5JjWdnViG7MgtokhyDz2/5Zmd2pTVUZm9OEfDbLugYr4cifzDAE2O8dDsmhSivwVrILijmmEYJHJMaT8uG8aXncBNjo2jdKIG4aA9RHk9pa1ckkixbtoxXX32VBx54gClTpuDx1PysemNt+Kcx1CSN3pf6omQAWG6Rj8z8YvbmFJKZX8x3O7M5kFdMfIyXb3Zm8/53e8JaV4zXQ7Q3eA7Y6zEczCsmOS6KVinxeD2G73Zlc3Hv1niNITUhmjaNE0mI8RLt9ZCWHEt8tJeUhGhio0pazAprkary+/1kZGTQs2dPADZt2sRxxx1X6fOMMeustdVex0YtfZFyiv0B9uYUsiOzgG0H8vEYgy8QYPPe3NCUqUDpoLFNe3IwxvDTvjyyCoppGB9NXpGfn/bn0TA+unR6VX7xkQ1NaZIUUxrEMVEejksLtoyz8ouJi/bStkkiOw7mc2LzZBKivcFzwh7D3uxCOjRPJjbaS2Gxn7aNEwlYS5PkWOKivER7DQ3io0tHN4tI+G3bto2xY8fyySef8O2339KmTZsqBf7RUOiLqwUCluJAgPwiPz/uyyOvKDjF6tud2fgCloytB9mbU8iW/XkUhRb38B/FKLIdZbq6M/MrnmbVODGGfblFdG3VkC37cuneJhWvgePTkmiREo/PH2BAhzQ6Nm9wxHWISN322muvMWHCBPLz83niiSdo3bp1WF5XoS/1Sn6Rny+3Z5JT6CudgrX9YD4/7MkhKTaajbuyWfPjfvKK/Ee0AlfZwI+L9tAkKbjkR482qUR7DH5rOSEtqbQr3Bvqyj6hWRI2tLhHTJQHr8eQmhBDlDe4jddjiPYG7xeRyGWt5YYbbuChhx6ie/fuvPDCC3To0CFsr6/QlzrH2uC0r8y8Ylb+sI+9OYXMeX8TALuzC6u9P2MgLspLfrGfmCgPfds1Ym9OEQ3iojjluMbERHk4oWkSLRvG07ZJQmjBFIWziNQ8Ywxer5c///nP3HfffcTFxYX19RX64phCn5/Ne3PZfjCfnZmF/Hwgj/U/HeCTTfsP+7yUhGiaJsdyTGoCHhNcbWtXdgHtGifSuWUDCn0Bjk9LpHvrVBonBRcSERFxirWWhQsX0r59e/r168f999/vWMNCoS81zucPsGV/Hpn5xfj8wXnf2w/m8+aXO9myL5ciX4ADVVhWNCk2ipzC4OInp7dPo23jBH7fvRVx0d4w/BYiIkcvOzuba665hv/7v/9j3Lhx9OvXz9GeRIW+VFtuoY8f9uSwdX8+WQXFrNm8n5SEGOZ9tPmI9tcqJZ5if4A2jRI4tnEi7ZokcFm/tiTHRddw5SIi4bN27VpGjRrF5s2bueuuu7jtttucLkmhLxXbnVXAi2u2UugLUFDs55WM7eyp5vn0JkkxpaueNUmKpUlSLAFrGd6lOa1S4mmREkdslFrtIuI+q1evpn///jRv3pwVK1bQv39/p0sCFPpCcNGYVz/bzptf7sAXsHywcW+Vnpd+TEMy84s5Pi0Jr8dwevsmNEyIYUinplqmVEQiUiAQwOPx0KtXL6ZOncq1115b4+vnHw29M0eg3EIfb365k3+t21rpoLkoj+HGYR2IjfIQG+2h3/FNOCY1XoPjRETKWbZsGTfccANLliyhZcuWTJ8+3emSfkOhHwECAcsLa7ay8OMfD7vmemyUh3O7tWRo56b0b59GUqz+PEREKlNcXMz06dO577776NixI1lZWbRs2dLpsiqkd3WXyin08ePeXJ7+YBOLP9te4TbDOjfjpFYNGd2nDWnJsWGuUESk/tu8eTOjR49m1apVXHnllTz00EMkJCQ4XdYhKfRdZkdmPlNe/KzCbvsebVK48vTjOLltI4W8iEgNmDlzJt988w0vvvgil1xyidPlVEpX2XOB5d/s4ukPNrPyh32/eaxhfDS/S2/JHed00vx2EZEakJeXx/79+znmmGM4ePAgBw4coF27dmGtQVfZiyC7swp45bPt/O3NDRxqefl/jOrO77u3Cm9hIiIu98UXXzBy5EgSExNZtWoVKSkppKSkOF1WlSn064l9OYVc9ew61m05UOHjzRvEMfaUNgzv0pz2zZLDXJ2IiLtZa3nyySeZMmUKqampPPzww3g89W8Wk0K/Diso9vPgsu/454pNFT5+XnpLurVqyOi+bTTSXkSklmRmZjJx4kT+85//MGLECBYsWEDTpk2dLuuIKCnqoEDA8sDSb3n8vR9+89iQTs2Yfm5n2jSuu6NDRUTcJCYmhi1btvDAAw8wZcqUetnCL6HQr0P2ZBcy49WveP2LHb+6v0/bRtzzh66c0DTJocpERCKL3+/n0Ucf5fLLLyc5OZlVq1YRFVX/I7P+/wb13L6cQq5b9Clf78jiYLkrz7VplMB/r+tPw3hdeEZEJFx+/vlnxo4dy4oVK4iLi2Py5MmuCHxQ6DvGWsvD73zPg8u++81jk/q34/azO+HxOHf5RRGRSPTf//6XiRMnUlBQwDPPPMP48eOdLqlGKfQd8OwnW5i2+Mtf3Te6TxumDG1P0+Q4h6oSEYlsjz32GNdeey09evRg0aJFdOjQwemSapxCP0yK/QGefO8H/t/6n/lxX17p/a1S4nntuv6kJsY4WJ2IiJx77rn89NNPzJw5k9hYd65aqhX5wiCroJgRD77P9syC0vv6tmvE42N60jjJnX9YIiJ1nbWWBQsW8Oabb7Jo0aJ6NSpfK/LVQdZa7l3yza/m2bdKiecfo7rTu23dub6yiEikycrK4pprruH5559n4MCB5OTk0KBBA6fLqnUK/VoSCFjOfviDX13Kdtq5nbmif3jXZxYRkV9bs2YNo0eP5scff+Suu+7itttuw+uNjGuTKPRryWXzV5cG/untm/DQyO7qyhcRcVhxcTGXXHIJgUCAFStWcNpppzldUlgp9GvY5r25XLlwLd/vzgHg3j90ZVSfNg5XJSIS2fbs2UNqairR0dH85z//4dhjjyU1NdXpssKu/oxaqOMyth6k7a2vc+YD75UG/jndWijwRUQc9vbbb9O1a1dmzpwJQPfu3SMy8EGhXyMWrPyR3z/20a/ue3xMTx67tKdDFYmISHFxMbfeeivDhw+nUaNGXHzxxU6X5Dh17x+lu177mrkfbi79ecZ5nbmsX1uM0Wp6IiJO2bx5M6NHj2bVqlVceeWVPPTQQyQk6EJlCv0j5A9YesxcSlaBD4BLeh/DrPO7EhOlzhMREacdOHCAzZs389JLL6mFX4ZC/wgEApaBD7xbGvhXnt6OO87p7HBVIiKRLTc3l8WLFzNmzBh69uzJ5s2b1bovR83SarLWctztb7B1fz4AQzs3U+CLiDjs888/p3fv3owbN46vvvoKQIFfAYV+NeQW+jj57mWlP9849ESeGl/tVRBFRKSGWGt5/PHH6dOnDwcPHuTtt9+mS5cuTpdVZ6l7vxomzF/N3pwiAMae0obrBrd3uCIRkcg2YcIEFi5cyFlnncUzzzxD06ZNnS6pTlPoV9HcDzez5scDAFzcKzhoT0REnDVs2DDS09O5/vrr69UFc5yi0K+C73fncNdrXwMwpFMz7r843eGKREQik9/v5+6776ZFixZceeWVjBkzxumS6hV9LKpEoc/PkNkrSn9+fIwW3BERccLPP//M4MGDufPOO1mzZo3T5dRLaukfhrWW0+5dXvrz0ilnaB6+iIgDXn31VSZOnEhhYSELFixg/PjxTpdULyn0D+OR5d+XDtx77NKenNgs2eGKREQiz4YNGzj//PPp3r07L7zwAieeeKLTJdVbCv1DyMwrZvbb3wEwrHMzzunWwuGKREQiS2ZmJg0bNqRTp0785z//YcSIEcTG6hLlR0N91YcwaWHwfFFctIdHLu3hcDUiIpHDWsszzzzDsccey8qVKwH4/e9/r8CvAQr9CjzyzsbS6XmPXdqT2CivwxWJiESGrKwsxowZw8SJE+nRowfHHnus0yW5ikK/nCJfgP8NdeunJkQzuFMzhysSEYkMa9asoUePHrz00kvMmjWLZcuW0apVK6fLchWd0y/n/1ZtKb296vYhDlYiIhJZ3n77bXw+HytWrOC0005zuhxXUku/nH+8sxGACf3aanqeiEgt27VrFx9//DEAt9xyCxkZGQr8WqSWfhlb9+dxMK8YgMtPa+dwNSIi7rZ06VLGjx9PTEwM33//PTExMaSkpDhdlqupKVvGqDmfANCzTQptGuuSjCIitaG4uJhbb72V4cOH07hxY15//XViYmKcLisiqKUf8u3ObLYdzAfgf4Zo4QcRkdqQlZXFsGHDWLVqFVdddRUPPvigrnsfRmrph/zh8Y8AOCY1ngEnpjlcjYiIOyUnJ3PSSSfx0ksv8c9//lOBH2YKfWDByh/JLfID8NT43g5XIyLiLrm5ufzpT39i48aNGGN4+umnufjii50uKyJFfPf+jsx87nz1KwBOatWATi0aOFyRiIh7fP7554wcOZJvv/2Wbt260b59e6dLimgR39J//N0fAGicGMO/ru7ncDUiIu5greWxxx6jT58+ZGZmsmzZMiZPnux0WREv7KFvjJlrjFlpjJl6iMdTjTFvGGM+MMY8WZu1+AOWF9dsBeDvF3UjLlrL7YqI1IR//vOfXHvttQwePJiMjAwGDRrkdElCmEPfGPMHwGut7Qe0NMZU1M8zDnjOWns6kGyMqbWT7G9+uYMif4AmSTEM6ti0tl5GRCRiFBYWAnDZZZcxd+5cXnvtNdLSNDi6rgh3S38g8FLo9nKgfwXb7AM6GGNSgNbAT7VVzMz/fg1A72MbYYyprZcREXE9v9/PX//6V7p37052djbx8fFcfvnlem+tY8Id+onAttDtLKCiq9l8CLQH/gx8Axwov4Ex5ipjzFpjzNo9e/YcUSGZecXszg5+Ir1+qAaWiIgcqZ9//plBgwYxY8YMevfWDKi6LNyhnwPEh24nHeL17wGuttbOJBj6E8tvYK2dY63tba3tfaTdRq9kBD97tGuSSMfmGrEvInIkXnnlFdLT01m3bh0LFizg2WefJTk52emy5BDCHfrr+KVLPx34sYJtEoCuxhgv0BewtVHIA299C6CFeEREjlAgEOD+++/n2GOPZf369YwfP97pkqQS4Z6nvxj4wBjTEjgLGGWMmWWtLTuS/2/AfOBY4GNgUU0X8f3ubLIKfABc1q9tTe9eRMTVvvnmGxo3bkxaWhr//ve/adiwIbGxsU6XJVUQ1pa+tTaL4GC+T4AzrbUZ5QIfa+1qa20Xa22StXaotTanpusYN3c1ALFRHto1Sazp3YuIuJK1lnnz5tGrVy9uuOEGAJo2barAr0fCPk/fWnvAWvuStXZnuF8bYPXm/ezILADgyXG9nChBRKTeyczM5NJLL+WKK66gb9++3HfffU6XJEcg4lbkm/N+cAW+89JbcmYHzc0XEanMV199Rc+ePXn55ZeZNWsWb7/9Ni1btnS6LDkCEbX2fm6hj3e/DU7x+8swXT5XRKQqmjZtSrNmzXj22Wfp10/LlddnEdXSf/2LHfgDluOaJHJsY53LFxE5lF27dnHzzTfj8/lIS0vjo48+UuC7QESF/j1vbACg3wmNHa5ERKTuWrp0Kd26deORRx5h/fr1AFpZzyUiJvSL/QEO5hUDcPlp7RyuRkSk7ikqKuLmm29m+PDhpKWlsWbNGvr06eN0WVKDIuac/urN+0tvH5eW5GAlIiJ108SJE3n++eeZPHkys2fPJiEhwemSpIZFTOivCoX+6e2bOFyJiEjd4vf78Xq93HjjjfzhD3/gwgsvdLokqSURE/qvfhZca/936ZpmIiICkJuby5///GdiYmJ44okn6NmzJz179nS6LKlFEXFOv6DYz9YD+QD0V0tfRISMjAx69+7N/PnzadSoEdbWymVOpI6JiND/fncO/oCldaN4WjSMr/wJIiIuZa3l0UcfpW/fvmRmZrJs2TLuvvtujc6PEBER+q99vgOA9GNSnC1ERMRh27Zt47bbbmPw4MFkZGQwaNAgp0uSMIqIc/pPrgguvavL6IpIpPryyy/p0qULxxxzDKtXr6Zjx45q3Ucg17f0v9uVXXr7rK4tHKxERCT8fD4fM2bMID09neeeew6ATp06KfAjlOtb+p9tPQgEW/lJsa7/dUVESm3dupUxY8bwwQcfMG7cOM4//3ynSxKHuT4F3/wieD6/T7tGDlciIhI+r7/+OuPHj6ewsJCFCxcybtw4p0uSOsD1of/jvjwA0pJjHa5ERCS82rVrx6JFi2jfvr3TpUgd4epz+oU+P5v35gJw6nG6yI6IuNuGDRuYP38+AOeccw6rVq1S4MuvuDr0l361C4CG8dG0bqQ1pEXEnay1zJ07l969e3PHHXeQk5MDgNfrdbgyqWtcHfrPfrwFgAt6tHK4EhGR2pGZmcno0aOZNGkSp5xyCmvXriUpSRcVk4q5+pz+ruwCADo0T3a4EhGRmldYWEifPn344YcfuOeee7j55pvVupfDcnXo5xb6Aeirkfsi4iLWWowxxMbGMmXKFLp160a/fv2cLkvqAdd27+/KKmBvTiFx0R6ObZzodDkiIjVi586dnHXWWbz55psAXH311Qp8qTLXhv66LQcASIiJwuvRylMiUv8tXbqU9PR0VqxYwd69e50uR+oh14b+F9syATg+Ta18EanfioqKuPnmmxk+fDhpaWmsXbtWi+3IEXFt6L/15U4ABnZo6nAlIiJH55VXXuH+++/n6quvZs2aNXTp0sXpkqSecu1Avp/2B1fi69yygcOViIgcmZ9++ok2bdpw0UUX8dFHH+ncvRw1V7b0i/0BfAELaCU+Eal/cnNzufzyy+nSpQubN2/GGKPAlxrhypb+D3tySm/HRWvOqojUH5999hmjRo3iu+++4/bbb6d169ZOlyQu4sqW/ptfBM/nj+jS3OFKRESq7tFHH6Vv375kZWWxbNkyZs2aRVSUK9tm4hBXhv7W0Pn81o3iHa5ERKTqvvjiC4YOHUpGRgaDBg1yuhxxIVd+hNydXQhAh+YaxCcidduKFSto0KABPXr04JFHHiE6OhpjtLaI1A5XtvQ//D64aMWxjXVlPRGpm3w+H3feeSeDBg1i6tSpAMTExCjwpVa5sqVfollynNMliIj8xtatWxkzZgwffPAB48eP59FHH3W6JIkQrgv9Qp+/9HbLFIW+iNQtX375JWeccQbFxcU8++yzjB071umSJIK4rnt/+8GC0ttRXtf9eiJSz3Xs2JGRI0eyfv16Bb6EnetSseRCO0mxruvEEJF6asOGDZx11lns3buXqKgonnjiCdq3b+90WRKBXBf6u7KCLf1+x2slPhFxlrWWuXPn0rt3b9atW8cPP/zgdEkS4VwX+qs37weg57GpDlciIpEsMzOT0aNHM2nSJE499VQyMjLo27ev02VJhHNd6GfmFwPQKkUL84iIc2666Sb+9a9/cc8997B06VJatGjhdEki7hu9vy83uDDP8WlJDlciIpEmEAiQmZlJamoqd999NxMnTuTUU091uiyRUq4KfWstW/fnA9C8oabriUj47Ny5k/Hjx5OXl8d7771HWloaaWlpTpcl8iuu6t4v6doHSE2IdrASEYkkb731Funp6aWL7Xi9urqn1E2uCv2SVr7HoKUsRaTWFRUVcdNNNzFixAiaNm3K2rVrueqqq/T+I3WWq0J/875cAIZ2buZwJSISCQoLC1m8eDFXX301q1evpkuXLk6XJHJYrjqn/8XPBwFoqjX3RaQWvfLKKwwbNozk5GTWrVtHgwa6oqfUD65q6R/IC57Tzy30OVyJiLhRTk4OEydO5Pzzz+exxx4DUOBLveKqlr7PHwCgjS6pKyI17LPPPmPkyJFs3LiRadOmcf311ztdkki1uSr0/Tb4vaUW5hGRGvTiiy8yfvx4mjRpwjvvvMOZZ57pdEkiR8RV3fs7DgZH76clxzpciYi4Sc+ePbngggvIyMhQ4Eu95q7QzwxebOe4JokOVyIi9d2KFSv485//jLWW9u3b88ILL9CkSROnyxI5Kq4K/fxiPwAJMa46ayEiYeTz+bjzzjsZNGgQS5YsYd++fU6XJFJjXBP61lqyQivyJccp9EWk+rZu3cqZZ57JzJkzGTduHOvXr1frXlzFNelY6AvgC1hiojzERWsJTBGpHr/fz5AhQ9i+fTvPPfccY8aMcbokkRrnmtDPCc3NT4hR4ItI1RUUFBAdHY3X62XOnDm0atWKE044wemyRGqFa7r3Sy620zBeF9oRkarZsGEDffr04f777wdgwIABCnxxNdeEfk5BsKWv8/kiUhlrLU8//TS9evVi586dpKenO12SSFiEPfSNMXONMSuNMVMr2e5xY8x5Vd3v/rwiAFITYo6yQhFxs8zMTEaPHs2VV15Jv379yMjI4KyzznK6LJGwCGvoG2P+AHittf2AlsaY9ofY7nSgubX2v1Xd99b9eQAaxCcih/X111+zePFi7rnnHpYuXUqLFi2cLkkkbMLd0h8IvBS6vRzoX34DY0w08BTwozHm91XdcZEvuO5+IGCPukgRcZdAIMDy5csBOPXUU/nxxx+57bbb8Hhcc4ZTpErC/RefCGwL3c4CKrrw/Xjga+DvQB9jzHXlNzDGXGWMWWuMWbtnzx4A9uYEu/d7tEmp+apFpN7asWMHw4YNY/Dgwaxbtw6A5s2bO1yViDPCHfo5QMnVcJIO8fo9gDnW2p3Ac8BvFrq21s6x1va21vZOS0sDYF9OIQCNk7TuvogELVmyhPT0dFauXMlTTz1Fz549nS5JxFHhDv11/NKlnw78WME23wPHhW73BrZUZce7s0Ohn6iBfCICU6dO5ayzzqJ58+asXbuWSZMmYYxxuiwRRx116BtjPKGBd1WxGBhnjJkNXAJ8ZYyZVW6bucCZxpj3gT8CD1Rlx9/szALU0heRoNatW/PHP/6RVatW0blzZ6fLEakTKp3UHhpY9xfgXiDOWpsfuj+OYHC/ALwFJFS2L2ttljFmIDAU+HuoCz+j3DbZwMXV+i34ZZ6+VuQTiVzPP/88Xq+XkSNHMnnyZKfLEalzqtrSvwm4Dphe5r7ngDtC+yiu6gtaaw9Ya18KBX6NiQ+FfeMkde+LRJqcnBwmTJjAmDFjWLBgAdZqFo9IRSpt6Vtri40xecAbwFpjzMdAe4LT73pZawuMMf7aLbPSGskubelrRT6RSPLpp58yatQoNm7cyLRp05g+fbrO3YscQlUTssha+70xZgrwE/ApsBr4vTHmpcM/tfblF/sp9AWIifKQFKvQF4kUmzZt4pRTTiEtLY3ly5czcOBAp0sSqdOqm5A7rLWfGWN6AA8DnYAPa76s6tkXmqOvkfsikcHn8xEVFcVxxx3HP/7xDy666CJd916kCqo8et8Y0wv4f8aYEcDJwCZgt7V2PeBoX1rJZXUbxOkKeyJu995773HiiSfy6aefAnD11Vcr8EWq6LChb4w5xRjzSujHz4G7CE6720dwhH1qaPpdvDFmdujrIWPMk7VZdHklc/T9Grwj4lo+n4/p06czaNAgoqOjtYSuyBGorHu/HRAd+voPMAO4leBc+gDBpXSPJ/jhoV3oOV4grhZqPaRAKOz9WndfxJV++uknxowZw4cffsiECRN45JFHSEpKcroskXrnsKFvrV0ELDLG/Eww4O8DLDAYeIXg3PwrgI3W2gtqudZD2nGwAIAOzZKdKkFEatH8+fPJyMjgueeeY8yYMU6XI1JvVbV/rMhaeylwAGgIFAAXAQ2AYwl+EHCMJzSiYHtmvpNliEgNys/P5+uvvwbg9ttv5/PPP1fgixyl6p4Ue5LgiP19BNfH722tXVfjVVVTybn8js3V0hdxg6+//pq+ffsybNgw8vPziY6Opm3btk6XJVLvVTX0Y40xjQguuRtN8BK5DwNNa6uw6ijyBQAtzCNS31lrmTNnDr1792bXrl08/fTTxMfHV/5EEamSqqy9H0Pw3P0IYJG19svQ/eOAhcaYfgQ/CDhm895cAKK9WoVLpL7Kz8/nsssu4+WXX2bIkCE8++yzuu69SA2rSks/AFxLsJV/a8md1tolwEOAnzCP1i+vcWLwynq7sgqdLENEjkJsbCyFhYXce++9vPXWWwp8kVpQlbX3fcD/hX7MLffY30xwketetVBblRX6gkv/d9A5fZF6JRAIMHv2bC655BLatGnD4sWLtW6+SC066tUtbNDnNVHMkSoMndOPjdJiHSL1xY4dOxg2bBg33XQTCxYsAFDgi9SyKqWkMSbWGPNvY0xs6OcmxpimxphEY4zfGJNYZtuFxpj+tVVwRfKKgsvwKvRF6oc333yT9PR0Vq5cyVNPPcXUqVOdLkkkIlS2DG/Jx+4A8PvQd4B5wFtAMcF19wtD2zcARgEtaqPYQyko1uh9kfrihRde4Oyzz6Z58+asXbuWSZMmqYUvEiaVNY1fMcb8zlpbDGCtLTbGXElwJP+N1tqi4N3WF9p+PMEFfBbXVsEVKZmyF6OWvkidZUPraZxzzjlMnz6dVatW0blzZ4erEoksh0xJY4wH+JTgMrzjQ/e1Bv4XuNlau7zc9nHA9cCdJR8SwqVkIJ+690Xqpueee47+/fuTn59PcnIyf/3rXzX/XsQBh0xJa23AWnsnwavpjQvd/TCwylr7UAVPuQfYAcyp6SIrk1cUDP34GG+4X1pEDiMnJ4cJEyYwbtw4PB4P2dnZTpckEtGqMmXvDeANY0wAuAXIgeD5fhvsrzPGmP8FLgBOsdYGDr232pFfHAx9ndMXqTs+/fRTRo0axffff8/06dOZNm0aUVH6NyripMP+CzTGvMUvc/MtcC/gCY3izzTGnBx67DzgVGvtrlqr9DA+/ekgAPHRaumL1AXWWv74xz+Sm5vL8uXLGTBggNMliQiVt/TXERqZD5xP8GI7LwIxwHbgI+AfwDHAdGPMn8sM6gubNo0S+Gl/npbhFXHY3r17iYqKIiUlheeff57k5GSaNGnidFkiEnLYkW/W2tuttX8lOHgPgpfSTQrd/6i19hGCPQA9gJOBp2q12kMoGciXFKeuQxGnvPvuu3Tr1o1rr70WgHbt2inwReqYSoe7G2P+BiwjGO6nA2OMMdeW3cZa+y3BefxnG2POq41CD6dkzf24KHXvi4Sbz+dj2rRpDB48mAYNGvCXv/zF6ZJE5BAqW5znBmAS8D8A1tpNwBjgb8aY40o2Cz22neA5/ztrrdpKaPS+SHht3bqVAQMGMGvWLCZMmMC6devo3r2702WJyCFU1tL/EjgXWA3Bufuh+fmvAQ9UsP0CoKsx5qQarbKKNE9fJLw8Hg87d+7k+eefZ968eSQmJlb+JBFxTGXn9Jdaa1cRHLhnCJ7Th2CL/nfGmA4AxhhvaPv9BBf0uaDWKv5NjcHvMV6PlvIUCYP8/HwefvhhAoEArVq14ptvvmH06NFOlyUiVVDVprElOEo/AGCtzQBOAbYAK/j1LIBFwDs1WGMlhQVTXyP3RWrfV199RZ8+ffif//kf3nvvPQCio6OdLUpEqqxKoW+tLbLWTrHWZpW5b621tsBae6a1trDM/f+w1q6sjWIrEgi19HNDq/KJSM2z1jJnzhxOPvlkdu/ezZIlSxg0aJDTZYlINdX7k+Al3ftNk2OdLUTExaZMmcLkyZM57bTTyMjIYPjw4U6XJCJHoNKJ7caYKKCFtXZrFbY9HrjXWntxTRRXFSVX7tIV9kRqz8UXX0yLFi246aab8Hj0b02kvqrKajbdgA+BhJI7jDFNgSUE19ovKrNtEsHL7oZNqKFPjFdvRCI1xe/3c99995GVlcW9997LaaedxmmnneZ0WSJylKqSlAVA+UvlFgPp5QIfoKiCbWtVIHRS3+PRQD6RmrB9+3aGDRvGHXfcwZYtWwgEwn4NLRGpJVUJfX/oq6xDvQuE/d2hpKW/eW/uYbcTkcq98cYbpKen8/HHH/P000/z/PPPqztfxEXq/WL1JVP2urdOcbYQkXpu9+7dXHTRRbRv354XXniBTp06OV2SiNSw+h/6ZRbnEZHq27VrF82aNaNp06YsWbKEPn36EBcX53RZIlILqpqUDY0xm0q+gAzAlL0vdP+y2iu1Yj5/MPWjtDiPSLU999xznHDCCSxatAiAM844Q4Ev4mJVbekXAH+twnYtgZuOvJzqK1l59+cD+eF8WZF6LTs7m2uvvZaFCxdy+umn079/f6dLEpEwqGroF1prF1S2UWgt/rCGfsk8/fRjGobzZUXqrfXr1zNq1Ch++OEHZsyYwR133EFUVL0/0yciVVDv/6UXBywGLc4jUlU//PAD+fn5vPvuu5xxxhlOlyMiYVTt0DfGTAJO57fT+ADC39wODeQ7mBfW5QFE6pU9e/bwySefcN5553HxxRdz9tln6zK4IhGoKqFv+PWAvwQglYpDP6kmiqqW0Dn9xkkxYX9pkfrg3XffZcyYMeTm5rJlyxZSUlIU+CIRqiqhHxf6AsBa+zDwcEUbGmM6AWG7wl6wnmDupyYo9EXK8vl8zJgxg3vuuYcTTzyRN954g5SUFKfLEhEHVRr61trPKBP6lYgB4o+moOoqGcgXrXn6IqWKi4sZNGgQH374IZdffjkPP/ywWvciUjOX1jXGdDPGeIEvgGY1sc+qyi8OnmWI0tr7IqWio6MZMWIEzz//PHPnzlXgiwhQhdA3xvQ1xhxyu1DYfwqkAV6gRc2VV7mSFn5OoS+cLytS5+Tn5/PHP/6R9957D4A77riD0aNHO1uUiNQpVWnpL+Iw3fvWWj/B0+qFwFhgWeiDQFiULMN7XJpaMhK5vvrqK/r06cMTTzzBqlWrnC5HROqoqgzkKwIKjTHTCYa7rWAbS3A0//XAv0IfBMIiv8hPHODVlcAkAllreeqpp7j++utJTk5myZIlDB8+3OmyRKSOqkrol1wudwrwOdAf+AQ4BdjIL1P3ugLHA4NruMbDio0Ohn1+kbr3JfK8+uqrTJ48maFDh7Jw4UKaN2/udEkiUodVp3lsgWEEW/t/CH2fDcwM3T4feNFau7eGazx8UaF+h7Tk2HC+rIijsrOzATjvvPN44YUXWLJkiQJfRCp1JH3ilt928VvgSeB/j7qiI2SMRu+L+/n9fu6++26OP/54fvrpJzweDyNHjsSj01siUgWH7d43xswFGgFnEByZX/pQBZvvsdZm1WBtVWJDnz+8Cn1xue3btzN27FjeffddRo8eTcOGusiUiFTPIUPfGBND8FK5ScAbBBfeqXtCfQ5ezdMXF3v99deZMGECeXl5zJs3jwkTJqh3S0Sq7ZB9gtbaImvtWcBPBIM/s5J9dTTGXFyTxVVFyXkGj0JfXOyFF16gZcuWrFu3jokTJyrwReSIVPUqe/YQ38saCkwAXj7Kmo6IuvfFbTZu3EggEKBDhw488cQTREVFERdX1RWxRUR+q6qjf0zoa1Xo+7LQ/XcA94ZuPwXEGGPOqtEKKxEIDd/XOCZxk2effZaePXty9dVXA5CUlKTAF5GjVp2W/qzQ7WfKPWYIjtovAB4EJgFvHmpHocGBnYA3rLWzDrNdM2CJtbbH4QorLA78UqFIPZednc2f/vQnnn32Wc444wwWLlzodEki4iJVCf0YIM5aW+F0PBM8ufi/BEf3LwTuNMZEW2uLK9j2D4DXWtvPGPO4Maa9tXbjIV73Aapwxb6YqGATXwP5pL7bvHkzw4YNY9OmTcyYMYOpU6fi9YZtRWsRiQBVCf3H+GXVvYrEEWztx1prdxpjBlUU+CEDgZdCt5cTXN3vN6FvjBkE5AI7KyuupIGfGFvVTguRuqlly5Z06tSJuXPncsYZZzhdjoi4UKVnwq21D1prCw/zeD7QDtgV+vnTw+wuEdgWup1FBZfhDU0VnA7ceqidGGOuMsasNcas9fmCy+/GRumkvtQ/e/bsYfLkyWRmZhIbG8urr76qwBeRWlMjSWmt3WKtrcpZ9Rx+6bJPOsTr3wo8Zq09eJjXm2Ot7W2t7e2zwW79kkvsitQXy5cvJz09nWeeeYZPPvnE6XJEJAKEOynXEezSB0gHfqxgmyHAn4wx7wHdjTFPH26HUaFz+TqnL/WFz+fjjjvuYMiQITRo0IDVq1fryngiEhbhPhG+GPjAGNMSOAsYZYyZZa2dWrKBtba0b9MY8561dlJVdqzufakvbr75Zh588EEuv/xyHn74YRITE50uSUQiRFhD31qbZYwZSHAhn79ba3cCGYfZfmCl+wx914p8UtcVFRURExPDjTfeSN++fRk5cqTTJYlIhAl789hae8Ba+1Io8Gtgh8FvUQp9qaPy8vKYPHky5557LoFAgFatWinwRcQR9b5PXC19qcu+/PJL+vTpw5w5c+jZsyeBQMDpkkQkgrlmcrvW3pe6xFrLP//5T6ZMmULDhg1ZunQpQ4cOdbosEYlw9b+lH5opqNH7Upfk5ORwzz33MGDAADIyMhT4IlIn1PuWfkn3vkJf6oJ169bRtWtXkpOT+eijj2jVqhUeXQ1KROoI17wbaSCfOMnv93P33XfTt29f7r//fgBat26twBeROqXet/RLGJ3TF4ds376dsWPH8u677zJ69Giuu+46p0sSEamQK0I/RgvziEOWL1/OyJEjycvLY968eUyYMEEfQEWkznJF6GvkvjglLS2NE044gfnz59OxY0enyxEROSxXNJF1Pl/C6bvvvuNvf/sbAF27dmXlypUKfBGpF1wR+l6vQl/CY+HChfTs2ZMHHniA7du3AxpPIiL1hztCX2+6Usuys7MZN24cl112Gb169SIjI4OWLVs6XZaISLW44px+kU9Lm0rtsdYyaNAg1q9fz4wZM5g6dSper9fpskREqs0VoZ9d6HO6BHGhQCCAMQZjDNOmTSMlJYUzzjij8ieKiNRRrujePyY13ukSxGV2797Nueeey6OPPgrA7373OwW+iNR7rgh9LcErNemdd94hPT2d5cuXExMT43Q5IiI1xhWh79FAPqkBxcXF3H777QwdOpTU1FRWr17N5MmTnS5LRKTGuCT0na5A3GDt2rXce++9XHHFFaxZs4Zu3bo5XZKISI1yxUA+tfTlaGzYsIFOnTpx6qmnkpGRQdeuXZ0uSUSkVriipa9z+nIk8vLymDx5MieddBKrVq0CUOCLiKu5oqWvFdGkur788ktGjRrFV199xS233ELPnj2dLklEpNa5IvS9ruivkHB5+umnue6662jYsCFLly5l6NChTpckIhIWrohLLcMr1ZGZmcmAAQPIyMhQ4ItIRHFF6GcVaEU+ObyPPvqIJUuWADBlyhTeeOMNmjVr5nBVIiLh5YrQzytS6EvF/H4/s2bNYsCAAUybNg1rLR6PB4/HFX/6IiLV4op3vjaNEpwuQeqgbdu2MWTIEKZNm8Yll1zCO++8o0GfIhLRXDGQz6A3cvm1bdu2kZ6eTn5+PvPnz+eyyy5T4ItIxHNH6Ou9XEKstRhjaNmyJddeey2jRo2iY8eOTpclIlInuKJ7X6EvAN999x1nnHEGGzZswBjDjBkzFPgiImW4I/TVvR/RrLUsWLCAnj178vXXX7N9+3anSxIRqZNcEfoaiB25srOzGTduHBMmTKB37958/vnnDB482OmyRETqJFfEpVr6kevBBx9k0aJFzJw5k3feeYdWrVo5XZKISJ2lgXxS7wQCAXbu3EnLli255ZZbGDFiBH369HG6LBGROs8dLX2lfsTYvXs35557Lqeddho5OTnExsYq8EVEqsgdLX2nC5CweOeddxg7diwHDhxg9uzZJCYmOl2SiEi94oqWvkep72o+n4/bb7+doUOHkpqayurVq/njH/+oHh4RkWpyRejrzd/djDGsXLmSSZMmsWbNGrp16+Z0SSIi9ZK696XO+ve//02/fv1o3rw5S5YsIS4uzumSRETqNbX0pc7Jy8vjqquu4sILL+T+++8HUOCLiNQAd7T0lfmu8cUXXzBq1Cg2bNjArbfeysyZM50uSUTENdwR+k4XIDViyZIlXHDBBTRs2JC33nqLoUOHOl2SiIiruKJ736Omviv06dOHUaNGkZGRocAXEakFrgh9ZX799eGHH3LRRRdRVFREo0aNmD9/Ps2aNXO6LBERV3JF6P+wJ8fpEqSa/H4/d911FwMGDOCzzz5j27ZtTpckIuJ6rgj949OSnC5BqmHbtm0MGTKE6dOnM2rUKNavX0+7du2cLktExPVcMZAv2uuKzy4RY/To0axfv55nnnmG8ePHa8qliEiYuCL0lRl1X2FhIX6/n4SEBJ588km8Xi8dOnRwuiwRkYjiiiayMr9u+/bbbznllFO47rrrAOjcubMCX0TEAe4IfTX16yRrLc888wy9evVi69atXHDBBU6XJCIS0dwR+k4XIL+RlZXF2LFjmThxIieffDIZGRmce+65TpclIhLRXBH6Sv26Z9++fSxZsoS77rqLZcuW0apVK6dLEhGJeO4YyKfUrxMCgQCLFy/mggsuoF27dvzwww+kpKQ4XZaIiIS4oqWvU/rO2717N+eccw4XXnghr7/+OoACX0SkjnFFS9+j0HfUsmXLGDduHAcOHODxxx/nnHPOcbokERGpgDta+ured8zf//53hg0bRmpqKmvWrOGaa67RbAoRkTrKHaGvjHFMjx49mDRpEmvXrqVr165OlyMiIofhiu59hX54vfTSS2zZsoWbbrqJoUOH6jK4IiL1hCta+pqzFx65ublceeWVjBw5kldeeQWfz+d0SSIiUg1hD31jzFxjzEpjzNRDPN7QGPOmMeZtY8x/jDExle+z5uuUX/v888/p3bs3c+fO5bbbbuPdd98lKsoVHUUiIhEjrKFvjPkD4LXW9gNaGmPaV7DZGGC2tXYosBMYUel+a7ZMKefAgQP079+fgwcP8vbbb3PPPfcQHR3tdFkiIlJN4W6qDQReCt1eDvQHNpbdwFr7eJkf04Dd5XdijLkKuAogpvkJaunXkvz8fOLj40lNTWXBggWcdtppNG3a1OmyRETkCIW7ez8R2Ba6nQU0O9SGxphTgVRr7SflH7PWzrHW9rbW9gbwKPVr3IcffkiHDh145ZVXALjgggsU+CIi9Vy4Qz8HiA/dTjrU6xtjGgGPAJdXZaeK/Jrj9/uZOXMmAwYMICYmRmvmi4i4SLhDfx3BLn2AdODH8huEBu69BNxmrd1SlZ1qMZia8fPPPzN48GDuvPNORo8ezfr16+ndu7fTZYmISA0Jd+gvBsYZY2YDlwBfGWNmldvmCqAXcIcx5j1jzMgw1xixli9fztq1a1mwYAHPPfccDRo0cLokERGpQcZaG94XNCYVGAq8b63debT7i23R3t4+ZzF3ntfl6IuLQIWFhaxfv55TTz0Vay07duygZcuWTpclIiKHYYxZVzKurTrCPk/fWnvAWvtSTQR+Ca29f2S+/fZbTjnlFIYOHcqePXswxijwRURczBUr8m07mOd0CfWKtZZnnnmGXr16sXXrVl544QXS0tKcLktERGqZK0K/ZUp85RsJEBydP27cOCZOnMjJJ59MRkYG5557rtNliYhIGLgi9GOiXPFrhIXX66Vp06bcddddLFu2TFPyREQiiCsWT9c5/cMLBALMnj2b008/nb59+zJ79mynSxIREQe4oomsafqHtmvXLs4++2xuuukmFi1a5HQ5IiLiIJe09KUiS5cuZfz48WRmZvLEE08wefJkp0sSEREHuSL0tfb+b73zzjsMHz6czp07s2zZMk466SSnSxIREYepe99l/H4/AAMHDuSBBx5gzZo1CnwREQHcEvpOF1BHvPjii3Tu3JmdO3fi9Xq58cYbSUhIcLosERGpI1wR+pHe1M/NzWXSpEmMGjWKRo0aUVxc7HRJIiJSB7ki9D0RnPmff/45vXv3Zt68edx+++28//77tG7d2umyRESkDnLFQL5Inqd/zz33kJmZydtvv83gwYOdLkdEROowd4R+hGX+/v37yc3NpXXr1jz++OP4/X6tnS8iIpVyRfd+JGX+Bx98QPfu3bn00kux1tKoUSMFvoiIVIk7Qj8CUt/v9zNz5kwGDhxIbGwsDz30ECYSfnEREakxLuned3f47d69m0suuYQVK1YwduxYHn/8cZKTk50uS0RE6hmXhL7TFdSuxMRE8vLyWLBgAePHj3e6HBERqafc0b3vwrP6BQUFzJo1i9zcXBITE/nkk08U+CIiclTcEfouy/xvvvmGU045hWnTpvHaa68B4PG44n+ViIg4yBVJ4pbFeay1zJs3j169erFt2zZee+01Ro4c6XRZIiLiEq4Ifbd078+aNYsrrriCvn37kpGRwTnnnON0SSIi4iIayFcHWGsxxjBmzBhiYmL4y1/+gtfrdbosERFxGVe09PfmFDldwhEJBALcf//9XHLJJVhrOe6447jlllsU+CIiUitcEfpNkmKcLqHadu3axdlnn83NN99MIBCgoKDA6ZJERMTlXBH6sVH169dYunQp6enprFixgieffJJ//etfxMfHO12WiIi4nCvO6dcneXl5TJgwgcaNG7Ns2TJOOukkp0sSEZEI4Y7Qrwcj+bZu3UrLli1JSEjgrbfe4vjjjychIcHpskREJILUr37xQ6jrkf/iiy9y0kkncd999wHQtWtXBb6IiISdK0K/rsrNzWXSpEmMGjWKLl26cOmllzpdkoiIRDBXhH5d7N3/4osv6N27N/PmzeP2229nxYoVtG3b1umyREQkgrninH5dXJEvPz+fvLw8li1bxqBBg5wuR0RExB0t/bpi3759zJs3D4A+ffqwceNGBb6IiNQZrgj9utC9//7779O9e3euueYafvzxRwBiYurfokEiIuJe7gh9B1/b5/MxY8YMzjzzTOLj4/n444917l5EROokV5zTd4q1lt/97ne8+eabjBs3jscee4zk5GSnyxIREamQK0Lfqe59YwyXXnopo0ePZty4cc4UISIiUkXuCP0wdvAXFBRw00030bNnTyZOnMjYsWPD9toiIiJHwxXn9MNlw4YN9O3bl0cffZTvv//e6XJERESqxRUt/dpu6FtrmT9/Ptdddx0JCQm8/vrrnH322bX7oiIiIjXMFS392u7cX7t2LVdccQV9+/YlIyNDgS8iIvWSK1r6ppZG8u3Zs4e0tDROPvlklixZwpAhQ/B6vbXyWiIiIrXNFS39mhYIBPj73/9O27ZtWbNmDQDDhw9X4IuISL3mjpZ+De5r165djB8/nqVLl3LhhRdywgkn1ODeRUREnOOKln5N9e4vXbqU9PR03n//fZ588klefvllUlNTa2bnIiIiDnNFS7+mfPzxxzRp0oR33nmHLl26OF2OiIhIjYr4lv6mTZv46KOPAJg6dSpr1qxR4IuIiCu5I/SP8Kz+okWL6N69O5MmTcLv9+P1eomPj6/h6kREROoGV4R+deXm5nL55Zdz6aWX0rVrV5YsWaKR+SIi4nquOKdfne79PXv2cPrpp/Pdd98xdepU7rzzTqKiXHEYREREDivi0q5JkyaceeaZPPHEE5x55plOlyMiIhI2EdG9v2/fPsaMGcOmTZswxijwRUQkIrki9A+3DO+KFStIT0/n5ZdfLl1dT0REJBK5I/QruM/n8zFjxgwGDRpEQkICn3zyCSNHjgx7bSIiInWFK0K/IrNnz+avf/0rY8eOZd26dfTs2dPpkkRERBzlioF8ZXv3c3JySEpK4k9/+hPHH388F154oXOFiYiI1CGuaOkbDAUFBVx77bX06dOH3NxcEhMTFfgiIiJlhD30jTFzjTErjTFTj2absrZu2kjfvn157LHHGDFihObdi4iIVCCsoW+M+QPgtdb2A1oaY9ofyTZl+fOyuH70CLZv387rr7/O7NmziY2NrZ1fQEREpB4Ld0t/IPBS6PZyoP8RblMqkJ9Jx269yMjI4Oyzz66hMkVERNwn3P3gicC20O0s4IQj2cYYcxVwVejHwozVH37ZqlWrGi5VymkC7HW6CJfTMa59Osa1T8c4PDocyZPCHfo5QMll7JKouKeh0m2stXOAOQDGmLXW2t41X6qUpeNc+3SMa5+Oce3TMQ4PY8zaI3leuLv31/FLd3068OMRbiMiIiLVFO6W/mLgA2NMS+AsYJQxZpa1duphtjklzDWKiIi4Ulhb+tbaLIID9T4BzrTWZpQL/Iq2yaxkt3NqoVT5LR3n2qdjXPt0jGufjnF4HNFxNtbami5ERERE6iBXrMgnIiIilas3oV8bK/nJr1V2/IwxDY0xbxpj3jbG/McYExPuGt2gqn+nxphmxphPw1WXm1TjGD9ujDkvXHW5SRXeL1KNMW8YYz4wxjwZ7vrcIvQ+8MFhHo82xrwW+n9xeWX7qxehXxsr+cmvVfH4jQFmW2uHAjuBEeGs0Q2q+Xf6AL9MX5UqquoxNsacDjS31v43rAW6QBWP8TjgOWvt6UCyMUbT+KrJGJMKLCC4fs2hXAesDf2/ONcYk3y4fdaL0KcWVvKT3xhIJcfPWvu4tfbt0I9pwO7wlOYqA6nC36kxZhCQS/DDlVTPQCo5xsaYaOAp4EdjzO/DV5prDKTyv+N9QAdjTArQGvgpLJW5ix8YSXChukMZyC//L1YCh/1wVV9Cv/wqfc2OcBs5tCofP2PMqUCqtfaTcBTmMpUe59Bpk+nArWGsy02q8rc8Hvga+DvQxxhzXZhqc4uqHOMPgfbAn4FvgAPhKc09rLVZVZjBVq3sqy+hXyMr+clhVen4GWMaAY8AlZ47kgpV5TjfCjxmrT0YrqJcpirHuAcwx1q7E3gOODNMtblFVY7xPcDV1tqZBEN/YphqizTVyr76Eoxaya/2VXr8Qi3Ql4DbrLVbwleaq1Tl73QI8CdjzHtAd2PM0+EpzTWqcoy/B44L3e4N6O+5eqpyjBOArsYYL9AX0Pzw2lGt7KsX8/SNMQ2AD4B3CK3kB1xcdmGfCrY5pQrdIhJSxWN8DcFP7xmhu56w1r4Y7lrrs6oc53Lbv2etHRi+Cuu/Kv4tJwPzCHaFRgMXWWu3VbA7qUAVj3EfYD5wLPAxcIG1NseBcuu9kveB0FifztbaR8s8dizwBrAM6Ecw+/yH3Fd9CH0oHcU4FHg/1CV3RNvIoen4hYeOc+3TMa59OsZ1R2jZ+v7AW5U1dutN6IuIiMjRqS/n9EVEROQoKfRFREQihEJfRGqdMcZrjDFO1yES6RT6Ii5hjDnfGNPvEI/FhqZO1dZrDzfG3Fjm578ZY5aW2WQ68N+q1mCMuTS0CJSI1KAopwsQkRozDVhujJlNcF50iduAtsBoY4wluIDHn6y1/wQwxvQACqh8HrUXiAO+sNYWlXssE7jdGNPUWnsLUAjkhfZ/NnAzcGn5qUSh5XA9QJH99ajicYAPOK/Mtobg9DoqeH0RqQKFvogLGGPaAF0JhmQf4Exr7XvGmGeAQmvt1cDVoW3fI7iKV4lPCIZ0oMx9MQQDvuya357Q/R0ILWZjjIkFDLAaOAf4RwUX/LgBuMZaW3JlRo+1tiD02EjgUSDfGFMYui+a4AcQnzHmxzL7iSa44MsDwN1VOjAi8isKfRF3uAxYZ63dHmrNV6a0xW2tjS3/oDFmAjDDWtu2kv3cB/xPuftKPyiUqWWwMWZ+6PYrwPmh24tC35dYa/eGnrMISAlt0xVYb60NhK7kdg7BZaBF5AjonL5IPWeMiQImEWytl3g3FLiXAXHGmGXGmGxjzEGCi3gc7lKd1TEDaAxEW2sNcAKwA/iOYLh3Al4leHrBQ7D3YHSZ58cDpwDfGWOGGmP+BTQH7gSmEFz1bUToOuFrgRahfYjIEVBLX6T+m0jwPH1ZZ1pr3yv5wRjzEFBsrfVVtANjzP8Aedbap6rzwmUvChRaIvT50Fc2wYva5BD8gPElcKO1dk655+cA1xpj5gBFwIUEewreJviB4AJr7RuhcQeXWWsXV6c+Efk1tfRF6jFjTGvgXuCxQzweZ4xpSrDFP84YMyH0dVK5TYcAZ5S7z2OMSSnz1cgYU9GlgNNDXfL/Bf5mrb2B4Ln/WGvtzwSXar0TeMwY899QPeXlAgcJducPIjhOYCxwjDGmJ8Frs5+gaX8iR0ehL1K/7QCmAuvL3V/SvZ8PtAbuB8YQPE8+Gzix3PZ+ypznD2lN8BroJV/7gLfKbmCMOTn02glAd2vtP0IPbQcyjTFeGzSb4IVZ2lDufccYMwJYBQwm2CvwCsHei/8HDCN4YZy2BGch/MsYk1DJMRGRQ1Doi9Rj1lqftfaJCh46M3SOPZ5gKL8BPGqtPZ9gN/qaKux+i7XWlHwRfL84pdw2XwJdrLW/t9ZuLPfYyZSZEWCtXUZwZkHpzAFjzO3As8B4YAPBq7ElExysl0lw6mFnoCfBS+B2Ay6pQu0iUgGd0xdxqVBXeEl3+DKCA+I+B3KttVuru7/QPPqCcne/CQw4TK974BCPGWOMh+Do/Zestd+H7uxL8Hrgc4FN1trrjTG9gO3W2h3GmG6hfXoPd/lQEamYQl/End4tc7sd8G+Cq+JZ4PEafJ2zCC6i4ytZXMcY0xH4FNgG/NdaO6Vk49A8/ZIpgl2BdUChMaa43H4TCYb7hDLPhV/WChgOrKjB30MkIij0RdypZHGeaEKBHFoW93KCU+JqhLU2v+zPoet6PwcsBP4KrAyF9e3W2vzQSnpFoedmcIj3IGPMYuBHa+31NVWriOicvohbePnl33NpkFpri4EkY8w0YASQAcwzxjQLXQSnuzGmE8Epfw2NMR1DLfUWQHTJz6GvzqHtTyj/4saYxsaYKcBnwNfAddba7cCpBM/tf2mMuc4Y07DWjoCIVEotfRF3iOOXRWtKF68JXYBnCcER8T0JjsJ/iGAwdyU4ar7suvuflNtv+Z+jQ/u7MLT/6wkuttOD4KC+q8rOpQ+dhx8IXEVwIZ/ZxpgXrbVjK/l9yn6IEZEaYn59jQsRcRtjTHNr7c5y9zUpWfb2KPd9GsE5/otD3fWH2zaW4JTBHdba9yvZ9m1gs7X2qqOtUUR+odAXERGJEOo+ExERiRAKfRERkQih0BcREYkQCn0REZEIodAXERGJEAp9ERGRCPH/ATwsZwpwGKO4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0, 1], [0, 1], 'k--')\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "    plt.xlabel('假正类率', fontsize=16)\n",
    "    plt.ylabel('真正类率', fontsize=16)\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "plot_roc_curve(fpr, tpr)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AUC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9578551433996665"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算曲线下面积AUC，虚线是随机分类0.5到1\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-21695.79268778, -18730.77517553, -77802.82337755, ...,\n",
       "         6435.01898138, -17346.21540113, -10936.13906559])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "是使用精度、召回率、 PR曲线，还是使用ROC，关键在于：正类非常少或者更关注假正类而不是假负类，选择PR曲线，反之ROC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练随机森林分类器，比较SGD分类器的ROC曲线和ROC AUC分数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "forest_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n",
    "y_probas_forest = cross_val_predict(forest_clf, X_train, y_train_5, cv=3,\n",
    "                                    method=\"predict_proba\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0.],\n",
       "       [1., 0.],\n",
       "       [1., 0.],\n",
       "       ...,\n",
       "       [1., 0.],\n",
       "       [1., 0.],\n",
       "       [1., 0.]])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_probas_forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 决策值,绘制ROC曲线，需要决策值不是概率，直接使用正类的概率作为决策值\n",
    "y_scores_forest = y_probas_forest[:,-1]\n",
    "fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_5,y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., ..., 0., 0., 0.])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_scores_forest "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAF6CAYAAAATeYHoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABcSUlEQVR4nO3dd3hU1dbA4d9OSCO00IsISBchEEJVqVIUUAGll4C0T8FruaAgCCJiwYsoKoh0UBQrKop0LPQAARQQFQFBOgaSQNrs7489qSQkgcmczMl6nydmZs+Zc9aMYdbsrrTWCCGEEML+vKwOQAghhBDuIUlfCCGEyCck6QshhBD5hCR9IYQQIp+QpC+EEELkE5L0hRBCiHxCkr4QQgiRT7g96SulyiilfrzO4z5KqW+UUpuVUoPdGZsQQghhZ25N+kqpIGAREHidw0YBO7XWzYHOSqnCbglOCCGEsDl31/QTgZ7Apesc0wpY7ry9GQjN5ZiEEEKIfKGAOy+mtb4EoJS63mGBwAnn7UtAmfQHKKWGAcMAAgMDG9aqVcu1gQq30sn/Sfrl/G+aspSDk4659nmZHJOmLKUgo/Pr1E/IMLbUZTpNeeprpJTptOfPKGadJtJMY069ZHbGMTvPc93XlYNj0pVdG3+qeHS6Y5LjTxdzhtfN+HUJkZWkTKKUwuEwt728nOUKHA7QDlPm7aWSyxLiTZmPDyhl/n5jY82TAvxTThwTA2gIDDTHAVy9qkhIAH8/8PVNKoO4OPD3Bz8/UCji4yE62lyjUKGUmC9eNNcrUVwlnzMqysRUtBh4e6WcMz5e4e9vrpOYEMeFf/4mPvYKvgWDiIu5eE5rXSqn75lbk342RQEBQCRQyHk/Da31HGAOQGhoqN65c6dbA8xr4hMdXIiO41xULOei4jh3OZbz0Sm3L8TEkejQOLTG4QCHNgnPoZ1l2nzwasj0mJTb5oPc4TDPcaQuT3++VI9ldM38SGVyW1zLS4GXMh+MSqnk+0llXqnKVHLZ9Y5Je56050z/mKuumXTu613TWealUKS/bgbHKJNUEuIhJkZRPMgkNC8Fx44pIv9V3HYbFA8y5wvfqfjzT6hbVxFcz5z3wgXFN19BiRKKbl0BzPNnzFDExcKY0Qo/P1P2yXLF3r3Qp5ciONjEsHEDLF2iaNNGMSjMnPP8efi/EYqSJWDu3JTYa9VUoGH3LkWxYubYJ/6j+PxzeOtN6NHDvL4li+HJJxVhA2DmTFP2229Qvz7UqK749ZeUc5YsCefPw5kzUMqZ9tq0gQ0bYPVqaNfOlD35JMyYAf/7Hzz1lCn7/HPo3h26djW3AeLjTWKvXRsiIlL+BidNggMH4IMPoIAzW377Lfz5JzRoAHfeacouXTJJulAhKFjQ9f8WoqOjue222wjw82HJogX07NkTpdTRGzlXXkz64cBdwKdAMLDV2nCscSUu0ZnETfI+n+p2+vKLMfFWh3vD1DUfbmk/QBXmQ+Z6x2T0O7MPdq7zQZ/Zea734a+ycc0bOSbNB30m8WR6jJf5DSkxX/M6vFKer8jgtXpdew3FtdfIKA5IeX7a2DOKP/Nj8rqYGPjjD1Ozq1Ejpfx//4OgIBicahjyZ5/B33+bZHPLLabstddgzx4YOBA6dDBle/bA5MlQr55JOAC//gp16kCVKibZJEl6i06dgjLO9tDu000i++QTeMh5zr1fwuf/gzL/B10HmrLdu2HwmyahzpmQcs5uy0zt9It3oLBzNNW7+2HTchj+IDS5zZTtuAJ7N0DzmlDP+XqOO+DAFqhQAWqkap+9NcjEXa4IlAgyZUH+EOgFxfyhpLMWXKEU1LoNKpaDQD9TVqIYdGwHFSum1IDBvGfR0ea9T/L003DXXea9SjJhgkn8QUEpZV27mtp+6j8xHx9TU08v6f9Bavfdd21ZkSLmx9Xi4+Px8fEhMDCQmTNn0qhRI6pUqXJT51RW7LKnlNqotW6llGoD3K61fjvVY5WAb4G1QHOgqdY6MbNzeUpNX2vNpasJnIuK5Xy6xH0uKpZzl83989Gmdh4dl+lLvoZSUCLQlxKBfpQs7EvJQn5pbhcv6ItvAa9MP9iTvuVfk9RSfbBneEwGH+wpCUShvNInulSPOeP2hA934X5am2bQ4sVTyvbuNYk2OBgCAkzZn3/C1q3QrJlJimCS66ZNUL48tG5tys6fNx/gRYvClCkp57z1VihdGjZuTGmCHTcOvvkGpk6Fzp1N2bffmuRx772m5gjw44/QooWp7f30kylzOMDbO+U1JGnVysS0YYO5DfD44zBzJjz/PLzwgilbswbat4e2bWHtWlMWE2NqoQC//w5Vq5rbISEmeR84AEk9nK+9Blu2wOjR0Ly5Kdu6Fd5+23wJadPGlJ09C8uWmVpy794pcX7+OSQmwoMPmkQIEB4O//xjarYVKpiyM2fg6FHz3lWqZMri400sPj6mxpzk8mUTv5dMEM+Rffv20atXLyZNmsTDDz98zeNKqXCtdY7HvFlS09dat3L+Xg+sT/fYUaVUO0xt//nrJXyrJTo0F6LjTFP65WsT+fk0t+OIS3Rk+9y+3l6UKGSSdslCvpQo5Jd8u2TS7cIm0RcP9DX9VULkERcvmgRQqVJKojhyxCSfVq2gSxdTduiQSViPPAJz55qyuDhTgwsKMsk66Xthz55w8CD88gvcfrspmzYNZs+G6dNNUgaTCPv1g06dUpJ+dLS5dsWKKUnf4YCEBJPUEhJSYj92DPbtM68hyeXL8Ntv5gtHkqJF4Y47Ur5sgElsPXqY15pat27mueXLp5T17Wtqh716pZQFB5tWgdKlU8oCAkwLQLVqKckfYNeua9/3MWOuLWva1PykVqqU+dKRXrdu15Y1bHhtWenSaWMEk+zr1bv22MIy/ypHtNbMnj2bp556imLFilE89TdfF8iLzftorU+SMoLf3dfm1KWrnL2cvibuTO6pbl+IjstR33SgrzclC/s5a+K+ybdTJ/KkRF/Ev4DUgoVbOBxmIBGkbaLcv9/U5jp1MvePHIE33zTNyW+8AeXKmfIRI2DFCpgzJyWZjxljkvjUqTB2rCn74QeTnA8eTDnuyhXze9kyeO89U0tO6ju9eNE8ntRHWq+eic/fPyXG+vVNTTs6OqXsllugTx/zWJLixeGtt9K+Pi8v+OgjM4Ar9UCrl14y8VesmFJ2773mS0zq59erZ74cpPfxx9eWZZRgmzQxP6mVLn1t4lUq7ZcNYV8XLlxg6NChfP7553Ts2JFFixZROv23q5uUJ5O+FbTWrD1whv+tPsTBU5ez/byggj7OWniqGrjzdvryAF/vXHwFwm4SEuDcOXO7bNmU8s2bTW20R4+UJtPPPjNJ+eGHU5pbV640fZqTJ6c0U69fb2rErVqZBA7medWqXdtnXLeu+X3pkqmtFStmmrHDw02TdFLSj4w0XwSiUg25bdTIDKg6fz6lrGFDCAtLWzOuVs00excvntIs7uV1bZ8rZJxMhw83P6k1aGAGXqVWqBCMGnXt81u0uLYs6f1LLbf6bIVIbe3atXz11Ve8/vrrPPnkk3jlQp+IJH3g59/PMe37Q+w5/i8ARfwLcEtQQVMTT66RJ/WZpyTy4oG++HhLR5XImagok1wDAqB6dVN2+jS8/rpJfkk14z/+ME3f1aubpuUkSSOGO3RIGaD03numP7hevZSkFR5umrq/+iol6UdFmb7xW29NOV+BAiYpBqZbMqtDB9N3u3u3SY5BQTBsmJk+lLqZeuZMM3gtdSvksGHmJ7U77oAFC9KWFSqUceKVRi6RXyQmJhIREUFISAg9evQgNDSU2267Ldeul6+TfvjRi7z+/SG2/GmqIyUL+fJoq2r0aXIr/j5SK8+vtE5JOrGxJml6e6dtdn39dZOoR482TbJ//21q1eHhZqBX0oCpmTPh2Wdh5Eh49VVT9t578N//ph2sFRlpzlmtWkrS9/Mz5y5RIm18999vBnSdPp2S9Lt3NzXz1Mm8UyfTd5s08AugZUvTP1ysWEpZpUqmzzq9VauuLUufyAFKlry2TAiRtRMnTtCvXz+2bt3KoUOHuPXWW3M14UM+Tfq/nIxk+urfWHfwDGBq9sNbViWseWUC/fLlW2JriYkmqSYmpszp3b7djFQOCICJE03ZwoWmqbhuXUiaEBIVZZrRixdPm/TnzjWD0AYPNom5VCkzennfPrhwIe21Y2LMHN4kHTqYpF+5ckpZ6dJm5HXqBF+5skns6a1YcW1Z+iZuMM3p6QdhFS0q/cNC5AXffPMNYWFhXLlyhVmzZlEx9SCSXGTJlD1XysmUvT/ORjF9zW+s3PsPAAV9vRl8ZxWGtriNogE+uRmmcJHUtfBffzXN3rVrQ82apmzZMlP7njoVBgwwZStWmClIAweaxA6mv7tzZzOv98MPzaCtEyfMILAaNUxCBzNALCzM9GnPn58Sx/vvm4Fmgwen1HQjIkxf9G23meQKZiR6XJxpEk9avUsIkX9prXnqqaeYMWMG9evX56OPPqJm0gdYDnjUlD13+/tiDG+uPcxnu/7GocG3gBf9mlTi0dZVKVnIL+sTCLfS2iRKpVISZb9+ZhDakiWmWRzM7VdeMaOtx40zZf/+a5L3m2+aEdwFCpjafMGCpm86SePGZpGNBx5IGS1erpxpEUg9xSgw0Cx0kt7QodeWZVSDlmQvhEhNKYW3tzePP/44r776Kv6pp6O4ga2T/plLV3l7w+8s236M+ESNt5eid6NbGNWmOuWLBVgdnsAsErJ9u+l3rlXLJPuuXc1iKNOmmWbwq1dN7fmDD8yxrVubEd516pj+7aTBcAD33GO+BLRokZLM27dPO6ULTHP8ypVpy7y8ZIS2EML1tNYsXryY6tWr07x5c6ZNm2bZlGzbDj0/ezmWjm/+yOItR0lwaB6sX551T7Xk5W71JOG7wV9/mZXATji3ToqLg8ceMwu1JC3CAqZpvXNns8AKmFpx//7mdqJzWSZ/fzOve+5ceOKJlGlq/fqZpvvUi1VVr25q/XfdlZuvTgghsufy5cv079+fsLAwZjs/6Kxcg8W2Sf+T8ONciI7jjgpFWPWfFszo1YDKJQOzfqLIUlRU2tXKFiwwc7+//97cT0gwzevNmsHSpabsxAkzYjwgwDTTJ80/r1wZQkPTzvG+7z4zmvyZZ1LKGjUyq7YFyPc1IYSH2LlzJw0aNGDZsmW8+OKLLEg/Z9UCtmzedzg0H+84DsBT7WpQs6ysA3kjFi82A+XatzfN5WfOmES8cKGpdQ8ebPrdDx8209QKFzYj0wsUMMukli6dsqhMlSpmQZeSJU2ffNLgtwceMD+pSRO7EMLTbd++nbvuuouyZcuyadMm7sojzY+2rOlv/fM8R8/HUK6oPy1ruHYJQ7vavNnM654509w/edLM037ppZS+75IlzSYfYL4QJLVQdesGgwaZpUqTDB1qppsNHJhS5udnWgQyWp9bCCHswOEwe6w0bNiQ8ePHs2fPnjyT8MGmSf8jZy3/4dCKshFNJhYtMokezPzy7dvN4LlPPzU19/LlzdzvAgXMYDkwfemDB5v++U2bUs4VGmqmszVr5v7XIYQQecXatWupX78+J0+exNvbm+eff97lG+bcLNsl/YvRcazafwqloEfoLVaHkyc4HGb+ekyMua+1mXs+frxZPtXHxwyeK1PGNOMnLbHasqX5QpC07CuYKWw+sqSBEEIki4+PZ+zYsbRv356EhAQuXbpkdUiZsl2f/ue7TxCX6KBFjVLcElTQ6nAsk5holo69fNkk8agos1nKnXeaZvnu3c2XgaQV20qUMJumCCGEyL4jR47Qu3dvtm3bxtChQ5kxYwYFC+bd3GOrpK+15qPtxwDo3cg9SxrmNZGRZvW5hg3NWu6FC8N335m577t3p9TaP/3U0jCFEMIWJk+ezMGDB/n444/p0aOH1eFkyVbN+7uOXeTwmShKFvKlbe0yVofjNtu2pSw+89tvZt/yb76BX34xZU2bmg1hRo60LkYhhLCLmJgY/v77bwDeeOMNdu/e7REJH2yW9D/abgbwdW94C74FbPXS0kjaLiE2Fpo0MUk9aYnZRo3MwLqJE82KdWAG4/nJasNCCHHT9u3bR2hoKF27dsXhcFCsWDGqVKlidVjZZpvMePlqPN84N9Lp1ejWLI72TBMmmP74QYPM/QIFYMQIc/uLL1KO27YNevd2f3xCCGFXWmtmzZpFo0aNuHjxIi+//DJeXp6XQj0v4kys3PsPV+ITaXpbcarYZOW9q1fNHuwXL5rafdGipm9+0SIzrS5pj/dLl8wofCGEEK4XGRlJ9+7defTRR2ndujURERHcc889Vod1Q2yT9PefjASg3e1lLY7ENbQ28+OffdZsIQtm85khQ+CPP1I2mSlaNO2ucEIIIVzL19eXo0eP8vrrr7Ny5UpKl/bcRd9sk/SPX7gCQKXieXeqRFZOn4a33za3lYJRo8ztTZtSVr+77z6z45wQQojck5iYyJtvvsnly5cJCAhg27ZtPP300x7ZpJ+ababs/X3RrDxzS3HP3JHl99/NHu///msWzilUyKxRHxVlFsQRQgjhHn///Tf9+vVj06ZN+Pv7M3z4cAoUsEe69OyvLE5aa/6+aGr6nrIgj9YweXLKWvfVqpk59Fqb5nuAggUl4QshhDt9/fXX1K9fn507d7Jw4UKGDRtmdUguZYuvLmcvxxKb4CCooA+F/DzjJfXqBcuXQ6tWULeu+f3557LErRBCWOWdd95h5MiRydvh1qxZ0+qQXM4WNf3jzlp+xTzcnx8fD1OmwI8/mvsff2y2oz11KmWjGkn4Qghhnc6dOzNmzBi2bNliy4QPNkn6yf35QXm3P3/QIDPP/osvzL70AP/5Dxw4IAvnCCGEFbTWLFy4kJ49e+JwOKhUqRKvvvoqfjb+ULZJ0nfW9PNYf77WKcvj9ukDTz1llsm18d+TEEJ4hEuXLtGvXz8GDRrEmTNniIqKsjokt/CMDvAsHL+Q92r6DgfUr29Wxhs71ky1a9XKDM4TQghhnR07dtC7d2/++usvXnzxRcaOHYu3t7fVYbmFPZJ+8nS9vJNRZ82CffvgxAkYOhRKlpSEL4QQVouPj6dHjx44HA42bdrEnUlbj+YTtkj6Kc371tf0Y2JMcn/sMbO97ZEjJuELIYSwztmzZwkKCsLHx4cvvviCSpUqERQUZHVYbmeLPv2T/1o/Rz8mBooXhwoVUsqaNpWNb4QQwmpr1qyhbt26TJ48GYD69evny4QPNkj68YkO4hM1JQv54e9jXZ9MbCzccYdZUe/LLy0LQwghhFN8fDzPPvssHTp0oHjx4jz88MNWh2Q5j2/ej0t0ANYN4rt82Wx4ExQEo0ebRXceeMCSUIQQQjgdOXKE3r17s23bNoYOHcqMGTMoKAOrPD/pO0zOp1hB969s07692fb24YfNDnhdurg9BCGEEBm4ePEiR44cYfny5VLDT8Xjm/cdWgMQYEHTfu/esHOn2ffewzdeEkIIjxcdHc0HH3wAQEhICEeOHJGEn47n1/SdSd+d/fkOh0nyDz5odsFL2gJXCCGENfbu3UvPnj05dOgQ9evXp06dOtKcnwGPr586c75bkn5UFFSsaFbWu3jR9ONLwhdCCOtorXn33Xdp3Lgx//77L2vWrKFOnTpWh5VneXzSd2fz/s8/mzn3b74J336b65cTQgiRhbCwMB577DHatGlDREQEbdu2tTqkPM0Gzfvmm4u/T+5/f+nQAVq0MFvg9u2b65cTQgiRhfbt2xMcHMwTTzyBlwyuypINkr7Gi9yt6R8+DCtXwhNPQECAJHwhhLBKYmIiL730EuXKlWPo0KH0lQ/kHPH4r0VJffoBvrmT9GNjoWZNeP11uHAhVy4hhBAiG/7++2/atm3LxIkT2bFjh9XheCSPT/q5PXr/4kV45hk4dQoiI3PlEkIIIbLw1VdfERwczM6dO1m0aBFz5syxOiSPJEk/C2XLwtSpppZfpUquXEIIIcR1HDhwgAcffJBKlSqxa9cuBgwYYHVIHsvjk35y876Lk/7Fi/D11+b8SkGRIi49vRBCiCxEOptXa9euzRdffMGWLVuoUaOGxVF5No9P+g6Hc8qer2tfyiuvwP33w333ufS0QgghsqC1ZuHChVSqVInNmzcD8MADD+Dn52dxZJ7P85N+0uI8BVxb0x8zxvx+7DGXnlYIIcR1XLp0ib59+zJo0CAaNGhApUqVrA7JVmyQ9J19+i4evV+iBKxeDZ07u/S0QgghMrFjxw4aNGjA8uXLmTJlCmvXrqVChQpWh2UrHj9P39V9+keOQPHiULQotGvnklMKIYTIhjVr1pCQkMCmTZu48847rQ7HluxT03dR0g8NhRo14KOPXHI6IYQQ13H69Gm2bNkCwDPPPENERIQk/Fxkm6Tvipr+1atQtSqcOQONG9/06YQQQlzH6tWrCQ4OpmfPnsTFxeHt7U2xYsWsDsvWbJD0zW9XJH1/f9i+3ST922676dMJIYTIQHx8PM8++ywdOnSgRIkSrFy5El9fX6vDyhds0Kdvsr7fTW64c/o0JCRAhQpQqpQrIhNCCJHepUuXaN++Pdu2bWPYsGG88cYbsu+9G3l8Td9Z0cevwM29lFq14MMPUwYGCiGEcL3ChQtzxx13sHz5ct577z1J+G7m8UkfwNtLoZS64ef/8gv8+6+Zm//3366LSwghBERHR/PYY49x+PBhlFLMnTuXhx9+2Oqw8iWPb94H8PG+8YQPUKcOnD1rdtKrWNFFQQkhhGDv3r307NmTQ4cOUa9ePapXr251SPmaLWr6Pl43/zJKljRL7wohhLh5WmveeecdGjduTGRkJGvXrmX48OFWh5XvuT3pK6XmKaU2K6XGZ/J4kFLqW6XUj0qp2dk5p88N9ucnJsJbb8HmzeBw3NAphBBCZOC9995j5MiRtG3bloiICNq0aWN1SAI3N+8rpboB3lrr5kqpd5VS1bXWh9Md1h9YqrX+UCn1gVIqVGu983rnLeB1Y837X3wB//kP3Hor/PXXDZ1CCCFEKrGxsfj5+TFw4EB8fX0ZNGjQTY25Eq7l7pp+K2C58/Z64K4MjjkP1FRKFQMqAseyOqmP9429jKpVYeRI6NrVbJ8rhBDixiQmJvLCCy9Qv359Ll++TEBAAIMHD5aEn8e4eyBfIHDCefsSUC2DY34COgGPAweBi+kPUEoNA4YB+JatdsMD+Ro0gJkzb+ipQgghnP7++2/69u3LDz/8QL9+/awOR1yHu2v6UUCA83ahTK4/FRihtZ6MSfqD0h+gtZ6jtQ7VWocCFLjBmr4QQoibs2LFCoKDgwkPD2fRokUsWbKEwoULWx2WyIS7s2U4KU36wcBfGRxTEKirlPIGmpCy/k6mbqRPv3ZtmDTJzM8XQgiRcw6Hg2nTplGpUiV27drFgAEDrA5JZMHdSf9LoL9SajrQA/hFKTUl3TEvA3OASKA4sCyrk/rmcPT+P//AwYPwwgtmfr4QQojsO3jwIGfPnsXLy4vPP/+cLVu2UKNGDavDEtng1qSvtb6EGcy3FWittY7QWo9Pd8x2rXUdrXUhrXU7rXVUVufNaU2/TBk4fBgmTABZJ0IIIbJHa838+fNp2LAhTz31FAClS5fGz8/P4shEdrm9M1xrfVFrvVxrfcpV58zp6H0vL6hWDSZPdlUEQghhb5GRkfTp04dHHnmEJk2a8Oqrr1odkrgBthgBl5Okv349HD0qG+sIIUR2/fLLL4SEhPDJJ58wZcoU1qxZQ/ny5a0OS9wAW6y9XyCbU/aioqB9e7MS319/QaVKuRuXEELYQenSpSlTpgxLliyhefPmVocjbkK+qun7+MC4cVCzpiR8IYS4ntOnTzNmzBgSEhIoVaoUP//8syR8G7BJ0s9eTd/Pz/Tjb9qUywEJIYQHW716NfXq1WPmzJns2rULQFbWswmbJP2cvYwyZXIpECGE8GBxcXGMGTOGDh06UKpUKXbs2EHjxo2tDku4kC2SfoFsbK178CAsXWqm6gkhhLjWoEGDmDZtGsOHD2f79u3ccccdVockXMwWST87zfvLlkH//jBtmhsCEkIID5KYmAjA008/zaeffsrs2bMpWLCgxVGJ3GCL0fvZad4PDoYBA6BLFzcEJIQQHiA6OprHH38cX19fZs2aRUhICCEhIVaHJXKRLWr62Zmy160bLFokSV8IIQAiIiIIDQ1lwYIFFC9eHC2Ll+QLtkj6WdX05W9ZCCEMrTVvv/02TZo0ITIykrVr1/LSSy/J6Px8wiZJ//p/rK+8AnPnSvIXQogTJ04wduxY2rZtS0REBG3atLE6JOFGtujT985i9P64ceDtbdbbb9XKPTEJIUResn//furUqcMtt9zC9u3bqVWrltTu8yFb1PSv92cbHw8LFsCdd0LDhm4LSQgh8oSEhAQmTZpEcHAwS5cuBaB27dqS8PMpW9T0r/e36+MDYWHQrx8UsMWrFUKI7Dl+/Dh9+/blxx9/pH///jz44INWhyQslm/SoCR8IUR+snLlSgYMGEBsbCyLFy+mf//+Vock8gBbpEJ1nQb+V1+FKlXgnnugeHE3BiWEEBarUqUKy5Yto3r16laHIvIIe/TpZ5LzY2Ph2WehZ0+IjHRvTEII4W4HDhxgwYIFAHTq1Ilt27ZJwhdp2CPpX+exmTMhNNTU9oUQwo601sybN4/Q0FCee+45oqKiAPD29rY4MpHX2CLpZ8bPD0aOhB07rI5ECCFyR2RkJL1792bIkCE0bdqUnTt3UqhQIavDEnmUPfr0ZeaJECIfio2NpXHjxvzxxx9MnTqVMWPGSO1eXJdNkn7GWX/uXAgKgk6dwN/fzUEJIUQu0VqjlMLPz48nn3ySevXq0bx5c6vDEh7Ats37J0/C0KFmO10fH6ujEUII1zh16hT33nsv3333HQAjRoyQhC+yzbZJXyl4+mlo184swSuEEJ5u9erVBAcHs2nTJs6dO2d1OMID2aR5/9qycuXMHH3p7xdCeLq4uDjGjx/PtGnTqFOnDuvXr6dOnTpWhyU8kC1q+pktzuPtDVnsxSOEEHneihUrmDZtGiNGjGDHjh2S8MUNs0VKzKg2/8Yb8MMPZsMdIYTwRMeOHQPgoYce4ueff2bWrFkEBARYHJXwZLZI+unFx8NTT0HLlpCQYHU0QgiRM9HR0QwePJg6depw5MgRlFIyWE+4hC2SfvqK/pUrZuR+lSogX4qFEJ5kz549NGzYkIULF/Kf//yHihUrWh2SsBFbDuQrUgTmzLEmFiGEuFFvv/02Tz/9NCVKlGDt2rW0adPG6pCEzdikpi9D9IUQnm/fvn20a9eOiIgISfgiV9iipp/e7t1mBb7bbjPr7wshRF61adMmihQpQoMGDZg5cyY+Pj6ZrjIqxM2yR00/3b+PkBC4/XbZaEcIkXclJCQwceJE2rRpw/jx4wHw9fWVhC9ylS2SfnotW5rft9xibRxCCJGR48eP06ZNGyZPnky/fv346KOPrA5J5BO2aN5P/81440Y4eBAqV7YkHCGEyNT+/ftp0aIF8fHxLFmyhH79+lkdkshHbFnTB6hVy+oIhBDiWrVq1aJnz57s2rVLEr5wO1sk/dT1/KNH4fhxy0IRQohrHDhwgHvvvZdz585RoEABZs2aRfXq1a0OS+RD9kj6qbL+hx9CpUowYoR18QghBJh97+fNm0doaCjh4eH88ccfVock8jl7JP1Uty9eBK3NdD0hhLBKZGQkvXv3ZsiQITRr1oyIiAiaNGlidVgin7NF0k/ttdfA4TBr7wshhFVGjx7Np59+ytSpU1m9ejXlypWzOiQh7Dl6XykoYItXJoTwJA6Hg8jISIKCgnjppZcYNGgQzZo1szosIZLZIjWmzvkJCZLwhRDud+rUKQYMGEBMTAwbN26kVKlSlCpVyuqwhEjDFs37STn/4kXw8YGOHS0NRwiRz3z//fcEBwfz448/MmDAALy9va0OSYgM2SLpJ/nzT/N7zRpr4xBC5A9xcXGMHj2ajh07Urp0aXbu3MmwYcNkKV2RZ9kj6Tv/gQUHwx9/wK5dFscjhMgXYmNj+fLLLxkxYgTbt2+nTp06VockxHXZovc76Tt1gQIyVU8IkftWrFhB+/btKVy4MOHh4RQpUsTqkITIFlvU9JNa0k6ehMuXrY1FCGFfUVFRDBo0iAcffJB33nkHQBK+8Ci2qOkn+fRTs9FO585w331WRyOEsJM9e/bQs2dPDh8+zIQJE3jiiSesDkmIHLNV0t+yBT76CEJDrY5ECGEnH3/8MQMGDKBkyZKsW7eO1q1bWx2SEDfEVkl/2DC4+27zI4QQrhISEkLXrl15++23KVmypNXhCHHDbNGnn6R1a3j0UZDNq4QQN2vTpk08/vjjaK2pXr06H330kSR84fFslfSFEOJmJSQkMHHiRNq0acOqVas4f/681SEJ4TK2Sfpaw9Kl8MUX5rYQQuTU8ePHad26NZMnT6Z///7s2rVLavfCVmzRp69QXL0K/fuDnx9cvWp1REIIT5OYmMg999zDyZMnWbp0KX379rU6JCFczhZJH8x2un36pN18RwghsnL16lV8fHzw9vZmzpw5VKhQgWrVqlkdlhC5wjbN+4GB8MEHpolfCCGy48CBAzRu3Jhp06YB0LJlS0n4wtZsk/SFECK7tNbMnTuXhg0bcurUKYKDg60OSQi3cHvSV0rNU0ptVkqNz+K4d5VSXbJ73pgYOHTINPMLIURmIiMj6d27N0OHDqV58+ZERERw7733Wh2WEG7h1qSvlOoGeGutmwPllVIZzqhXSt0NlNVaf53dcy9YALVqQbduLgpWCGFLv/76K19++SVTp05l9erVlCtXzuqQhHAbd9f0WwHLnbfXA3elP0Ap5QO8D/yllHogOydVCmrUgCJFIDHRVaEKIezC4XCwfv16AJo1a8Zff/3F2LFj8fKSHk6Rv7j7Lz4QOOG8fQkok8ExA4BfgdeAxkqpUekPUEoNU0rtVErtTCpr1w7OnoV3382FqIUQHuuff/6hffv2tG3blvDwcADKli1rcVRCWMPdST8KCHDeLpTJ9RsAc7TWp4ClwDU7W2it52itQ7XWabbW8fWFihVdHLEQwmOtWrWK4OBgNm/ezPvvv09ISIjVIQlhKXcn/XBSmvSDgb8yOOZ34Dbn7VDgaO6HJYSwm/Hjx3PvvfdStmxZdu7cyZAhQ1CykIfI52466SulvJwD77LjS6C/Umo60AP4RSk1Jd0x84DWSqkfgEeB17Nz4pYtoUkTOHAgm5EIIWytYsWKPProo2zbto3bb7/d6nCEyBOyXJHPObDuv8ArgL/W+oqz3B+TuD8CvgcKZnUurfUlpVQroB3wmrMJPyLdMZeBh3P0KoDwcIiOhvj4nD5TCGEXH374Id7e3vTs2ZPhw4dbHY4QeU52a/qjgVHA86nKlgLPOc+R7VSrtb6otV7uTPguoYA9e2DRIqhZ01VnFUJ4iqioKMLCwujbty+LFi1Cy65bQmQoy5q+1jpeKRUDfAvsVEptAapjpt811FpfVUpZPlGuWjXzI4TIX3bv3k2vXr04fPgwEyZM4Pnnn5e+eyEykd0Nd+K01r8rpZ4EjgG7ge3AA0qp5dd/qhBC5I4///yTpk2bUqpUKdavX0+rVq2sDkmIPC2nA/n+0VrvAUoCb2Hm0pd3dVA5dfkyPPkkTJ9udSRCCHdISEgA4LbbbuPNN99kz549kvCFyIZsJ32lVEPgM6VUR6AR8CdwRmu9C9OtbpnISzBjBsyfb2UUQgh32LhxIzVq1GD37t0AjBgxgpIlS1oclRCe4brN+0qppsBY5929wIuYaXd9MSPsI53T7wKcv8F8kfDXWo/IlYgzUKSIqeUXKeKuKwoh3C0hIYHJkyczZcoUqlevLkvoCnEDsurTrwL4OH++ACYBz2Lm0jswS+lWxST6Ks7neAP+uRBrpooWgWFPuvOKQgh3OnbsGH379uWnn34iLCyMmTNnUqhQIavDEsLjXDfpa62XAcuUUn9jEvyrgAbaAiswc/MfAQ5rrbvmcqxCiHxqwYIFREREsHTpUvr27Wt1OEJ4rOy2j8VprfsAF4GiwFXgIaAIUAnzRcAy58/DypXwyy9WRiGEcKUrV67w66+/AjBu3Dj27t0rCV+Im5TTTrHZQG3gPGZ9/FCtdbjLo8qhffugc2d46y2rIxFCuMKvv/5KkyZNaN++PVeuXMHHx4fKlStbHZYQHi+7Sd9PKVUcs+SuD2aL3LeA0rkVWE4EBUHHjlC3rtWRCCFuhtaaOXPmEBoayunTp5k7dy4BAQFZP1EIkS3ZWXvfF9N33xFYprXe7yzvDyxWSjXHfBGwTHAwTB1iZQRCiJt15coVBg4cyCeffMI999zDkiVLZN97IVwsOzV9BzASU8t/NqlQa70KmAEk4ubR+kII+/Hz8yM2NpZXXnmF77//XhK+ELkgO2vvJwAfOO9Gp3vsZWUWuW6YC7Flm3YotAZZblsIz+JwOJg+fTo9evTg1ltv5csvv5R184XIRTe9uoU29roimBu1/BPw8jKr8gkhPMM///xD+/btGT16NIsWLQKQhC9ELstW0ldK+SmlPldK+Tnvl1RKlVZKBSqlEpVSgamOXayUuiu3As5IvHNjX19fd15VCHGjvvvuO4KDg9m8eTPvv/8+48ePtzokIfKF6yZ9lfK12wE84PwNMB/4HojHrLsf6zy+CNALKJcbwWamfz9ITIThw915VSHEjfjoo4+47777KFu2LDt37mTIkCFSwxfCTbKq6a9QSt2vtY4H0FrHK6WGYkbyP621jjPFOsF5/ADMAj5f5lbAmfHyAm9vd19VCJFdWps1vDp16sTzzz/Ptm3buP322y2OSoj8JdOkr5TyAnZjluEd4CyrCPwPGKO1Xp/ueH/gCWBi0pcEIYQAWLp0KXfddRdXrlyhcOHCvPDCCzL/XggLZJr0tdYOrfVEzG56/Z3FbwHbtNYzMnjKVOAfYI6rg8zK8uXQpYtZmU8IkXdERUURFhZG//798fLy4vLly1aHJES+luVAPq31t1rrdpi++2eAgZCmv18ppf4HdAUe0lo7Mj5T7vntMHzzDURFufvKQojM7N69m4YNG7JkyRKef/55NmzYQOnSeWIRTyHyrevO01dKfU/K3HwNvAJ4OUfxRyqlGjkf6wI001qfzrVIr6NPb7h9MNSqZcXVhRDpaa159NFHiY6OZv369bRs2dLqkIQQZL04TzjOkfnAg5jNdj4GfIGTwM/Am8AtwPNKqcdTDepzm9uqwv2h7r6qECK9c+fOUaBAAYoVK8aHH35I4cKFKVmypNVhCSGcrtu8r7Uep7V+ATN4D8xWuoWc5W9rrWdiWgAaAI2A93M1WiFEnrVhwwbq1avHyJEjAahSpYokfCHymCz79JVSLwNrMcn9bqCvUmpk6mO01ocw8/jvU0p1yY1Ar2fRQpg9G2JjszxUCOFiCQkJTJgwgbZt21KkSBH++9//Wh2SECITKmnubIYPKvUUMBboDGzWWnsrpdoAK4BgrfWfSimH1trLefyTQF+ttdsa2/3KVdc+JdcTvb8iMTEgs4CEcJ/jx4/Tq1cvNm/ezKBBg5g5cyaBgYFZP1EIcVOUUuE3kmuzqunvxyT87c6LeDnn538DvJ7B8YuAukqpO3IayM3o2BH69wd/2etPCLfy8vLi1KlTfPjhh8yfP18SvhB53HUH8mmtV4NZex8zZa8I8C9mFH+4Uqqm83FvrXWi1vqCUmo3Zvre/twMPLWePeFhGcgnhFtcuXKF999/n5EjR1KhQgUOHjyIj4+P1WEJIbIhu7vsacwofQeA1joCaAocBTaR9svDMmCdC2MUQuQRv/zyC40bN+Y///kPGzduBJCEL4QHyVbS11rHaa2f1FpfSlW2U2t9VWvdWmsdm6r8Ta315twINjMnT8KRI+68ohD5i9aaOXPm0KhRI86cOcOqVato06aN1WEJIXIouzX9PO255+Aut27mK0T+8uSTTzJ8+HDuvPNOIiIi6NChg9UhCSFuQFaL86CUKgCU01ofz8axVYFXtNYPuyK47CpXDsoXd+cVhchfHn74YcqVK8fo0aPx8rJFXUGIfOm6U/YAlFIhwE9a64KpykoDq4Cmzu11k8qDnccWzqV4r+FXrrr+4JsNPNTwFnddUgjbS0xM5NVXX+XSpUu88sorVocjhEgnt6bsAVwF0m+VG4+Zpx+Xrjwug2OFEB7k5MmTtG/fnueee46jR4/icLh9Dy0hRC7JTtJPdP6kltmngHw6COHBvv32W4KDg9myZQtz587lww8/lOZ8IWzEFv+a//tf6NPH6iiE8GxnzpzhoYceonz58oSHh/PII4+QsoO2EMIOshzI5wnOnYNT8tkkxA05ffo0ZcqUoXTp0qxatYrGjRvjL8tbCmFL2a3pF1VK/Zn0A0QAKnWZs3xt7oWaudenwaJFVlxZCM+2dOlSqlWrxrJlywBo0aKFJHwhbCy7Nf2rwAvZOK48MPrGw7kxpUpBxYruvqoQnuvy5cuMHDmSxYsXc/fdd3OXLHQhRL6Q3aQfq7XOsi7tXIvf7UlfCJF9u3btolevXvzxxx9MmjSJ5557jgIFbNHTJ4TIgi3+pb/5FiTcZzbeEUJc3x9//MGVK1fYsGEDLVq0sDocIYQbZWdxnprAFq11cef9IcDdXDuND6Ao0DrpWHfwK1dd+5TYQLvqt/DFF+66qhCe5ezZs2zdupUuXboAEB0dLdvgCuHBbnRxnuzU9BVpB/wVBILIOOkXymkArtCzJ3QLseLKQuR9GzZsoG/fvkRHR3P06FGKFSsmCV+IfCo7Sd/f+QOA1vot4K2MDlRK1QbcusMeQKdO0EmSvhBpJCQkMGnSJKZOnUqNGjX49ttvKVasmNVhCSEslGXS11rvIVXSz4IvEHAzAd2IhAR3X1GIvC0+Pp42bdrw008/MXjwYN566y2p3QshXLMin1KqnlLKG9gHlHHFOXMiPBz+/NPdVxUi7/Lx8aFjx458+OGHzJs3TxK+EALIRtJXSjVRSmV6nDPZ7wZKAd5AOdeFlz3vvAOzZ7v7qkLkLVeuXOHRRx9l48aNADz33HP07t3b2qCEEHlKdmr6y7hO877WOhEz2C8W6AesdX4RcJtGjaBdO3deUYi85ZdffqFx48bMmjWLbdu2WR2OECKPys5AvjggVin1PCa5ZzTHT2NG8z8BfOr8IuA2jz8O7Rq484pC5A1aa95//32eeOIJChcuzKpVq+jQoYPVYQkh8qjsJP2k7XKfBPYCdwFbgabAYVKm7tUFqgJtXRyjECITX331FcOHD6ddu3YsXryYsmXLWh2SECIPy8lAPg20x9T2uzl/TwcmO28/CHystT7n4hizdPUqJLq1bUEIa12+fBmALl268NFHH7Fq1SpJ+EKILN3I6H3NtU38GpgN/O+mI7oBQ4fAN99YcWUh3CsxMZGXXnqJqlWrcuzYMby8vOjZsydeXi6ZiCOEsLnrNu8rpeYBxYEWmJH5yQ9lcPhZrfUlF8aWbb5+4ONjxZWFcJ+TJ0/Sr18/NmzYQO/evSlatKjVIQkhPEymSV8p5YvZKrcQ8C1m4Z08af48xX0ykE/Y2MqVKwkLCyMmJob58+cTFhaGUhl99xZCiMxl2iaotY7TWt8LHMMk/sgszlVLKfWwK4MTQhgfffQR5cuXJzw8nEGDBknCF0LckOxurasz+Z1aOyAM+OQmYxJCAIcPH8bhcFCzZk1mzZpFgQIF8PfP7orYQghxreyO/lHOn23O32ud5c8Brzhvvw/4KqXudWmE2TB5MuzZ4+6rCpF7lixZQkhICCNGjACgUKFCkvCFEDctJzX9Kc7bC9M9pjCj9q8CbwBDgO8yO5FzcGBt4Fut9ZTrHFcGWKW1zrK3/rff4MKFrI4SIu+7fPkyjz32GEuWLKFFixYsXrzY6pCEEDaSnaTvC/hrrTOcjqdM5+L/MKP7FwMTlVI+Wuv4DI7tBnhrrZsrpd5VSlXXWh/O5Lqvk80d+wYOhDvuyM6RQuRdR44coX379vz5559MmjSJ8ePH4+3t1hWthRA2l52k/w4pq+5lxB9T2/fTWp9SSrXJKOE7tQKWO2+vx6zud03SV0q1AaKBU9mIj/btoXTp7BwpRN5Vvnx5ateuzbx582jRooXV4QghbCjLPn2t9Rta69jrPH4FqAKcdt7ffZ3TBQInnLcvkcE2vM6pgs8Dz2Z2EqXUMKXUTqXUzqziFyIvO3v2LMOHDycyMhI/Pz+++uorSfhCiFzjkmW8tNZHtdYZjehPL4qUJvtCmVz/WeAdrfW/17neHK11qNY6FMxqfJFZTSgUIo9Zv349wcHBLFy4kK1bt1odjhAiH3D32p3hmCZ9gGDgrwyOuQd4TCm1EaivlJqb1Uk//BAuXnRViELkroSEBJ577jnuueceihQpwvbt22VnPCGEW2R39L6rfAn8qJQqD9wL9FJKTdFaj086QGud3LaplNqotR6S1Uk7dYLixXMjXCFcb8yYMbzxxhsMHjyYt956i8DAQKtDEkLkEyp7rfIuvKBSQZiFfH7QWmdroN71+JWrrpd/t5EH6le4+eCEyEVxcXH4+vpy4sQJfvrpJ3r27Gl1SEIID6WUCk/q4s4Jt2/NpbW+qLVe7oqEL4QniImJYfjw4XTu3BmHw0GFChUk4QshLGGL/ThPnIDE600qFMIi+/fvp3HjxsyZM4eQkBAcDofVIQkh8jFbJP3Ro+HKFaujECKF1prZs2fTqFEjzp07x+rVq3nllVcoUMDdw2iEECKFLZJ++fIgn6UiL4mKimLq1Km0bNmSiIgI2rVrZ3VIQgjh9tH7ueJ//1PIXiQiLwgPD6du3boULlyYn3/+mQoVKuDlZYvv1kIIG5BPIyFcIDExkZdeeokmTZowbdo0ACpWrCgJXwiRp9iipi+ElU6ePEm/fv3YsGEDvXv3ZtSoUVaHJIQQGbJFNeTpp6yOQORXSUvpbtu2jfnz5/PBBx9QpEgRq8MSQogM2aKmf/681RGI/KpUqVJUq1aNBQsWUKtWLavDEUKI67JFTd/ZhSqEW/z222+8/PLLANStW5fNmzdLwhdCeARbJP3Spa2OQOQXixcvJiQkhNdff52TJ08CoJSyOCohhMgeWyR9IXLb5cuX6d+/PwMHDqRhw4ZERERQvnx5q8MSQogcsUXSX7LE6giEnWmtadOmDR9++CGTJk1i/fr13HLLLVaHJYQQOeb2XfZcza9cdV2g2CaiD0itS7iWw+FAKYVSiq+++opixYrRokWLrJ8ohBC5zGN22csNjz5qdQTCbs6cOUPnzp15++23Abj//vsl4QshPJ4tkn7LllZHIOxk3bp1BAcHs379enx9fa0ORwghXMYWSV8IV4iPj2fcuHG0a9eOoKAgtm/fzvDhw60OSwghXMYWSf/gQasjEHawc+dOXnnlFR555BF27NhBvXr1rA5JCCFcyhYD+SrX2cShtTKQT9yYAwcOULt2bQD27dtH3bp1LY5ICCGuL18P5KtV2+oIhCeKiYlh+PDh3HHHHWzbtg1AEr4QwtZssfb+sKGyIprImf3799OrVy9++eUXnnnmGUJCQqwOSQghcp0tkr4QOTF37lxGjRpF0aJFWb16Ne3atbM6JCGEcAtbNO8LkRORkZG0bNmSiIgISfhCiHzFFgP5Kt3+A7+tK2d1KCIP+/nnn7l8+TIdO3bE4XAA4OUl33mFEJ4pXw/ki4mxOgKRVyUmJjJlyhRatmzJhAkT0Frj5eUlCV8IkS/Z4pNv9myrIxB50YkTJ7jnnnuYMGECPXr0YN26dbINrhAiX7PFQD6ptIn0Tpw4QXBwMFeuXGHBggUMHDhQEr4QIt+zRdIXIonWGqUU5cuXZ+TIkfTq1YtatWpZHZYQQuQJtqgjL1tmdQQiL/jtt99o0aIFBw4cQCnFpEmTJOELIUQqtkj6hw5ZHYGwktaaRYsWERISwq+//srJkyetDkkIIfIkWyT93r2sjkBY5fLly/Tv35+wsDBCQ0PZu3cvbdu2tTosIYTIk2yR9KUFN/964403WLZsGZMnT2bdunVUqFDB6pCEECLPkoF8wuM4HA5OnTpF+fLleeaZZ+jYsSONGze2OiwhhMjzbFHT373H6giEu5w5c4bOnTtz5513EhUVhZ+fnyR8IYTIJlsk/ZUrrY5AuMO6desIDg5m/fr1jB49msDAQKtDEkIIj2KLpF+/vtURiNyUkJDAuHHjaNeuHUFBQWzfvp1HH31UFtsRQogcskXS79zJ6ghEblJKsXnzZoYMGcKOHTuoV6+e1SEJIYRHkoF8Is/6/PPPad68OWXLlmXVqlX4+/tbHZIQQng0W9T0o6OtjkC4UkxMDMOGDaN79+5MmzYNQBK+EEK4gC2S/tvvWB2BcJV9+/bRqFEj5s6dy7PPPssrr7xidUhCCGEbtmjeDyxodQTCFVatWkXXrl0pWrQo33//Pe3atbM6JCGEsBVb1PRHjrQ6AuEKjRs3plevXkREREjCF0KIXGCLpC88108//cRDDz1EXFwcxYsXZ8GCBZQpU8bqsIQQwpYk6QtLJCYm8uKLL9KyZUv27NnDiRMnrA5JCCFszxZJ/4MPrY5A5MSJEye45557eP755+nVqxe7du2iSpUqVoclhBC2Z4uBfGfOWB2ByInevXuza9cuFi5cyIABA2RlPSGEcBNbJP0+va2OQGQlNjaWxMREChYsyOzZs/H29qZmzZpWhyWEEPmKLZr3ZdxX3nbo0CGaNm3KqFGjALj99tsl4QshhAVskfRF3qS1ZuHChTRs2JDjx4/TtWtXq0MSQoh8zRZJf9t2qyMQ6V26dIl+/foxaNAgGjVqREREBJ07d7Y6LCGEyNdskfS3S9LPc86fP8+qVat48cUXWbt2LRUqVLA6JCGEyPdsMZCvcWOrIxAADoeDL7/8kq5du1KlShX++OMPihUrZnVYQgghnGxR028iSd9yZ86coVOnTnTv3p2VK1cCSMIXQog8xhY1fWGttWvX0r9/fy5evMi7775Lp06drA5JCCFEBmxR0z971uoI8q/XXnuN9u3bExQUxI4dO/i///s/WWxHCCHyKFsk/U8/szqC/KtBgwYMGTKEnTt3UrduXavDEUIIcR22aN4vVcrqCPKX5cuXc/ToUUaPHk27du1kG1whhPAQtqjpP9Td6gjyh+joaIYOHUrPnj1ZsWIFCQkJVockhBAiB9ye9JVS85RSm5VS4zN5vKhS6jul1Bql1BdKKV93xyiutXfvXkJDQ5k3bx5jx45lw4YNFChgi4YiIYTIN9ya9JVS3QBvrXVzoLxSqnoGh/UFpmut2wGngI7ujFFc6+LFi9x11138+++/rFmzhqlTp+Lj42N1WEIIIXLI3TX9VsBy5+31wF3pD9Bav6u1XuO8Wwq4ZuNcpdQwpdROpdROgA8+yJ1g87srV64AEBQUxKJFi4iIiKBt27YWRyWEEOJGuTvpBwInnLcvAZnuj6eUagYEaa23pn9Maz1Hax2qtQ4FiLmSG6Hmbz/99BM1a9ZkxYoVAHTt2pXSpUtbHJUQQoib4e6kHwUEOG8Xyuz6SqniwExgcHZO2qe3S2ITQGJiIpMnT6Zly5b4+vrKmvlCCGEj7k764aQ06QcDf6U/wDlwbzkwVmt9NDsnDQx0VXj5299//03btm2ZOHEivXv3ZteuXYSGhlodlhBCCBdxd9L/EuivlJoO9AB+UUpNSXfMI0BD4Dml1EalVE83x5hvrV+/np07d7Jo0SKWLl1KkSJFrA5JCCGECymttXsvqFQQ0A74QWt96mbP51euuh4z6QdeHF7u5oPLh2JjY9m1axfNmjVDa80///xD+fLlrQ5LCCHEdSilwpPGteWE2+fpa60vaq2XuyLhJzn0m6vOlL8cOnSIpk2b0q5dO86ePYtSShK+EELYmC1W5KtWzeoIPIvWmoULF9KwYUOOHz/ORx99RClZy1gIIWzPFkm/WVOrI/AciYmJ9O/fn0GDBtGoUSMiIiLo3Lmz1WEJIYRwA1usoyqrwWaft7c3pUuX5sUXX2Ts2LF4e3tbHZIQec6lS5c4c+YM8fHxVoci8hkfHx9Kly6dawOpbZEuj/8NyK6umXI4HEyfPp27776bJk2aMH36dKtDEiLPunTpEqdPn6ZChQoEBASglLI6JJFPaK25cuUKJ06YNexyI/Hbonn/h01WR5B3nT59mvvuu4/Ro0ezbNkyq8MRIs87c+YMFSpUoGDBgpLwhVsppShYsCAVKlTgzJlrVqB3CVvU9G+paHUEedPq1asZMGAAkZGRzJo1i+HDh1sdkhB5Xnx8PAEBAVkfKEQuCQgIyLWuJVsk/ZYtrI4g71m3bh0dOnTg9ttvZ+3atdxxxx1WhySEx5AavrBSbv792aJ5X6RITEwEoFWrVrz++uvs2LFDEr4QQgjAJknfvWsK5l0ff/wxt99+O6dOncLb25unn36aggULWh2WEMJi586do0+fPgQFBVG6dGkmTJiQ/NjVq1cZMWIERYsWpUyZMkydOjX5sUmTJqGUwsvLi1KlStGjRw8OHTpkxUsQLmKLpL98udURWCs6OpohQ4bQq1cvihcvLtOMhBBp9OzZk5MnT/LZZ58xduxYXn75ZT7++GMAHn/8cVauXMnSpUuZPHkyL7zwAp999lnyc8uVK8fWrVt588032bdvH82bN+fYsWNWvRRxk2zRp5+fu9/27t1Lz549OXToEOPGjWPSpEn4+PhYHZYQIo/466+/WL9+Pbt27aJBgwa0adOGH3/8kcWLF9OiRQvmz5/P0qVL6dKlCwBbtmxh5syZdO/eHQBfX18aN25M48aNadOmDTVq1ODll19m1qxZVr4scYNskfR7PGx1BNaZOnUqkZGRrFmzhrZt21odjhAij7lw4QJgmviTvPbaa0RGRrJu3ToSExNp165d8mMNGjTg22+/zfBcZcuWpUuXLpk+LvI+WzTv5zcXLlzg+PHjALz77rtERERIwhdCZKhOnTpUrFiRsLAwPv/8c7TWVKtWjYYNG3Lw4EEKFy5MiRIlko8fOHAgGzZsyPR89erV49ixY1y5csUd4QsXk6TvYX788Ufq169Pnz590FpTvHhx2SxHCJEpPz8/vv76a/z8/OjevTuhoaFs2bIFMLX/9Ku+FStWjDp16mR6vqCgIAD+/fffXItZ5B5bJP3Nm62OIPclJiYyefJkWrVqhZ+fHzNmzJC5xEK4kVLXjh/q0sWUff11StmcOaZs2LCUspMnTVn6nasbNjTl4eEpZZMmmbJJk1LKUj9+I4KDgzl48CDvvvsuJ0+epFWrVqxcuZL4+Hi8vEwa2Lp1K0qp5J/MyOeOZ7NF0j912uoIcteZM2do27YtEydOpE+fPuzatYuGDRtaHZYQwoP4+vryf//3f+zbt4/atWszfPhwAgMDiY6OBkyz/e7du3n//feve56LFy8CULRo0VyPWbieLZJ+s2ZWR5C7AgMDiYmJYdGiRSxZsoTChQtbHZIQ+Y7W5ie1r782Zc6B74Cp4WttavxJypc3ZSdPpn1+eLgpT/0dftIkU5a6pn8z3/Hff/99OnbsmHy/ZMmSTJgwgRMnTlCiRAkuXLhAZGQkBQsWpH79+pQtW/a659u/fz+VK1eWNUA8lC2Sfrnr/416pKtXrzJlyhSio6MJDAxk69atDBgwwOqwhBAext/fn3Xr1qXpgz9//jwBAQF069YNgK9T9U/s2bMn03OdPXuWr776igcffDCXohW5zRZT9uzm4MGD9OrVi4iICKpXr07Pnj2T+92EECInunTpQlBQEA899BDPPvssZ86cYeLEiQwbNoy6devy8MMPM3LkSAC8vb2v2Xo7Li6OHTt28McffzBlyhQKFy7MuHHjrHgpwgVskUn+Omp1BK6htWb+/Pk0bNiQEydO8M0339CzZ0+rwxJCeLBixYqxdu1aHA4H3bp1Y+zYsQwYMIBXX30VgIULF/Lwww8zYsQInnnmGR577LE0z//nn39o0qQJTzzxBI0bN2bbtm0yY8iDKZ2+k8rD+JWrrgeP+oFZ48pZHcpNe/HFF3n++edp3bo1S5cupXz6ob5CiFx34MABateubXUYIp/L6u9QKRWutQ7N6Xlt0bxfuZLVEdwcrTVKKfr27Yuvry///e9/8fb2tjosIYQQNmOL5v169ayO4MY4HA6mTZtGjx490Fpz22238cwzz0jCF0IIkStskfQ90enTp7nvvvsYM2YMDoeDq1evWh2SEEIIm7NF0o/xsCWgV69eTXBwMJs2bWL27Nl8+umnBAQEWB2WEEIIm7NF0v/hB6sjyL6YmBjCwsIoUaIEO3bsYPjw4bKspRBCCLewxUA+T6gkHz9+nPLly1OwYEG+//57qlatKitaCSGEcCtb1PRbtrA6guv7+OOPueOOO5LnxdatW1cSvhBCCLezRdLPq6KjoxkyZAi9evWiTp069OnTx+qQhBBC5GOS9HPJvn37CA0NZf78+YwbN45NmzZRuXJlq8MSQgiRj9ki6W/eYnUE17py5QoxMTGsXbuWl156CR8fH6tDEkLkMwsXLkQphVIKLy8vKlWqxH//+9/k7XRz65ruquD89ddfya8v/c/ChQvdEoOnscVAvugoqyMwzp8/z4oVKxg8eDCNGzfm8OHD+Pr6Wh2WECKf27FjB3FxcWzfvp0JEyZw+vRplixZYnVYLjN79mwaptt/uEqVKhZFc609e/awceNGnnjiCatDsUfSb9rU6gjghx9+oG/fvpw5c4Y2bdpQuXJlSfhCiDwhNNQs0d68eXOio6OZPHkyc+fOxc/Pz+LIXKNmzZrJrzEv2rNnDzNmzMgTSd8WzfuFC1t37YSEBCZNmkTr1q0JCAhgy5Yt0ncvhMizQkJCiIuL4/z581aHIixgi6RvFa01999/Py+88AJ9+/YlPDyckJAQq8MSQohMnT59GqUUJUqUAODEiRM8+OCDFC1alLJly/Lkk0/icDiAlD7zPXv28NBDD1GoUCFq1arFli0pA6l+/fVX7rzzTvz9/WnWrBlHjhxJc72LFy/Sr18/ChcuTNmyZXnhhRdI2t21VatWDB8+nEaNGlG8eHFWrlxJs2bNKFasGF9++aVLXm9sbCyjRo0iKCiIoKAgRo0aRWxsbPLjGzduRClFYmIiL774IpUrV07T9REfH8+YMWMoU6YMxYsXJywsjEuXLiU/funSJQYOHEipUqUoVqwY3bp14+zZswBMmjQJpRSDBg3i6NGjyeMNJk2a5JLXdkO01h7941u2mp75xUltlSVLlujFixdbdn0hhGv9+uuvVofgMgsWLNDmY97Yv3+/rlmzpr7nnnuSy1q2bKlDQkL02rVr9aeffqqLFy+uFyxYoLXW+siRIxrQd9xxhx45cqRes2aNDgkJ0fXr19daax0fH6+rV6+umzVrpletWqUnT56sfXx8dKVKlZLP3759e12jRg29YsUKPXv2bF2oUCE9derU5GsXLlxYf/bZZzo4OFgXKFBAL1iwQHfo0EHfe++9Wb6+pPg2bNiQ6TFDhw7VZcuW1cuWLdPLli3TZcqU0cOGDUt+fMOGDRrQw4YN040aNdJvvPGGPnDgQPLjzzzzjC5Tpoz++OOP9TfffKOrVq2qe/bsmfz4qFGjdPny5fWKFSv0V199pevWrauHDBmitdb6xIkTeseOHXrixIm6XLlyeseOHXrHjh36xIkTWb62rP4OgZ36BnKmLfr0jx5137WuXr3K6NGjCQkJYdCgQfTr1899FxdCWKLysyutDgGAv17pdEPPS73Ud0hICPPmzQNMpa9Pnz60aNGCWrVqkZCQwDvvvMO2bdsICwtLfk7t2rWZOXMmAOPGjaNXr16A2Ufk8OHDfPfdd1StWpUOHTqwe/dudu3aBcDPP//M6tWr2b17N/Xr1wfMUuQTJkzgqaeeAqB3795069aNFStWUKZMGcLCwjhy5AibNm3K9utr3bp1mvtHjhyhcuXKHDt2jHnz5vHZZ5/x4IMPAuDn58dDDz3Ec889x6233pr8nAMHDvDTTz+lGYt15coVZsyYwXvvvUePHj0AOHfuHEOHDuXq1av4+/tz7NgxgoODuf/++wGoXr06Fy5cAKB8+fKUL1+e/fv34+vrmyfGHdiief+2qu65zoEDB2jSpAlvv/02v//+u3suKoQQN2n37t2sXLkSpRRjxoxJTnZKKXr06MG3335Lp06dKFOmDBs3buTKlbS7mA0bNiz5dokSJUhISADg8OHDFC9enKpVUz6EW7RIWSJ19+7dFC1aNDnhg2nSj46OTv4MLVeuXHIsqW/nxPvvv8/u3buTf8qXLw/A3r17cTgctGrVKs31HQ4H+/btS3OO//3vf9cMvv7999+JjY0lLCwsuWk+LCyM+Ph4jh07BsAjjzzCunXraN68OWPGjOHEiRM0b948R/G7ky1q+pUr5e75tdYsWLCAUaNGUbBgQVauXMl9992XuxcVQuQZN1rDzivq169P/fr1uf/++3nttdfo2bMnAJcvXyYkJITSpUvTp08fJkyYwKxZs655fmbT3xwOB15eaeuO3t7eae6nT+BJ97WzX98VqlWrluaLRXqpY8js+o0aNbrmeUnHfPLJJ1SrVi3NY0lfnLp06cKhQ4dYtWoVmzZt4t577+XRRx9lxowZN/JScp0tavq5befOnTzyyCM0adKEiIgISfhCCI80btw4du3axZo1awBYt24dR44c4bvvvuPxxx+nadOmGbZipk/kSapWrcr58+eTa71gmvST1K9fn3///ZeIiIjksk2bNhEYGEj16tVd9bIyVa9ePby8vNi4cWOa63t5eVGvXr0sn1+tWjV8fX25evVq8henwMBAXn/9dS5evAjAa6+9xvHjxxkxYgTLli1j8uTJzJ8/P815/P39k1tHrGaLmn5kZO6c9+zZs5QqVYpGjRqxatUq7rnnnkz/+IUQIq9r3Lgxbdu25dVXX6Vdu3bJI/gXLFhA3bp1effdd9m8eXO2F7bp2LEjt956K/3792f8+PGEh4fz2WefUaFCBQDuuusu2rVrR69evXjttdc4deoUzz//PM8995xb1gi49dZbeeSRR/i///u/5C6Lp556iiFDhqTpz89MwYIFefLJJxk9ejRaaypUqMCkSZO4ePEiZcuWBeDQoUMsW7aMl156iYCAAL766qtr3r+QkBBOnTrFvHnzqFWrFj/++CPPPvus619wdtzI6L+89ONbtpoe85ZrR+8nJibqV199VRcsWFBv377dpecWQuRtdh69r7XW69ev14DesWOH1lrr5557TpcoUUIXKVJEDxw4UA8fPlxXq1ZNJyQkJI+OP3LkSPLzk0a7Jzlw4IBu1aqVLliwoA4JCdHPPPNMmtH7Fy5c0H369NGBgYG6dOnSeuLEiToxMVFrbUbvT5w4UWut9cCBA/XAgQO11lpPnDhRt2zZMsvXl53R+1evXtUjR47UxYoV08WKFdMjR47UV69ezfT1pBcXF6dHjx6tS5UqpQsXLqwfeOABffTo0TSvLywsTJcuXVoXLFhQt2jRQu/du/ea88yZM0ffcsstukCBAvqOO+7I8rXl1uh9pV3Yr2IFv3LV9fOv/sBzA8q55HynT59mwIABrF69mu7du/P+++8TFBTkknMLIfK+AwcOULt2bavDEPlcVn+HSqlwrXWOpwPYok8/pIFrzrN69WqCg4P54YcfmD17Np988okkfCGEELZhiz59V9myZQslS5Zk3bp11KlTx+pwhBBCCJeyRU3/Zvz555/Jo03Hjx/Pjh07JOELIYSwJVsk/c1bsj4mI8uWLaN+/foMGTKExMREvL29CQgIcG1wQgghRB5hi6Sf0+mP0dHRDB48mD59+lC3bl1WrVolU/GEEMk8fYCz8Gy5+fdniz79Zs2yf+zZs2e5++67+e233xg/fjwTJ06kQAFbvA1CCBcoUKAACQkJ+Pj4WB2KyKcSEhJyLS/ZItv55OBVlCxZktatWzNr1qxrNmkQQgh/f3+ioqJk5o6wzOXLl/H398+Vc9uieT8r58+fp2/fvvz5558opSThCyEyVapUKc6ePUtMTIw08wu30loTExPDuXPnKFWqVK5cwxY1/cOH4d66GT+2adMm+vbty5kzZ7j//vu57bbb3BucEMKj+Pv7U6ZMGU6dOkVsbKzV4Yh8xs/PjzJlyuRaTd8WSf/M2WvLEhISmDJlCi+++CJVq1Zl69athISEuD84IYTHKVq0KEWLFrU6DCFczhbN++l2PARg+vTpvPDCC/Tr14/w8HBJ+EIIIfI9W9T0y5ROuR0VFUWhQoV47LHHqFq1Kt27d7cuMCGEECIPsUVNH+Dq1auMHDmSxo0bEx0dTWBgoCR8IYQQIhW3J32l1Dyl1Gal1PibOSa1vbsO06RJE9555x06duwo8+6FEEKIDLg16SulugHeWuvmQHmlVPUbOSa1xJhLTBjWkZMnT7Jy5UqmT5+On59f7rwAIYQQwoO5u6bfCljuvL0euOsGj0nmuBJJ5RoNiYiI4L777nNRmEIIIYT9uLsdPBA44bx9Cchg3H3WxyilhgHDnHdjD+//aX+FChVcHKpIpyRwzuogbE7e49wn73Huk/fYPWreyJPcnfSjgKRt7AqRcUtDlsdorecAcwCUUju11qGuD1WkJu9z7pP3OPfJe5z75D12D6XUzht5nrub98NJaa4PBv66wWOEEEIIkUPurul/CfyolCoP3Av0UkpN0VqPv84xTd0coxBCCGFLbq3pa60vYQbqbQVaa60j0iX8jI6JzOK0c3IhVHEteZ9zn7zHuU/e49wn77F73ND7rGQXKSGEECJ/sM2KfEIIIYS4Po9J+rmxkp9IK6v3TylVVCn1nVJqjVLqC6WUr7tjtIPs/p0qpcoopXa7Ky47ycF7/K5Sqou74rKTbHxeBCmlvlVK/aiUmu3u+OzC+Tnw43Ue91FKfeP8fzE4q/N5RNLPjZX8RFrZfP/6AtO11u2AU0BHd8ZoBzn8O32dlOmrIpuy+x4rpe4Gymqtv3ZrgDaQzfe4P7BUa303UFgpJdP4ckgpFQQswqxfk5lRwE7n/4vOSqnC1zunRyR9cmElP3GNVmTx/mmt39Var3HeLQWccU9ottKKbPydKqXaANGYL1ciZ1qRxXuslPIB3gf+Uko94L7QbKMVWf8dnwdqKqWKARWBY26JzF4SgZ6Yheoy04qU/xebget+ufKUpJ9+lb4yN3iMyFy23z+lVDMgSGu91R2B2UyW77Oz2+R54Fk3xmUn2flbHgD8CrwGNFZKjXJTbHaRnff4J6A68DhwELjontDsQ2t9KRsz2HKU+zwl6btkJT9xXdl6/5RSxYGZQJZ9RyJD2XmfnwXe0Vr/666gbCY773EDYI7W+hSwFGjtptjsIjvv8VRghNZ6MibpD3JTbPlNjnKfpyRGWckv92X5/jlroMuBsVrro+4LzVay83d6D/CYUmojUF8pNdc9odlGdt7j34HbnLdDAfl7zpnsvMcFgbpKKW+gCSDzw3NHjnKfR8zTV0oVAX4E1uFcyQ94OPXCPhkc0zQbzSLCKZvv8f9hvr1HOItmaa0/dnesniw773O64zdqrVu5L0LPl82/5cLAfExTqA/wkNb6RAanExnI5nvcGFgAVAK2AF211lEWhOvxkj4HnGN9btdav53qsUrAt8BaoDkm9yVmei5PSPqQPIqxHfCDs0nuho4RmZP3zz3kfc598h7nPnmP8w7nsvV3Ad9nVdn1mKQvhBBCiJvjKX36QgghhLhJkvSFEEKIfEKSvhAi1ymlvJVSyuo4hMjvJOkLYRNKqQeVUs0zeczPOXUqt67dQSn1dKr7LyulVqc65Hng6+zGoJTq41wESgjhQgWsDkAI4TITgPVKqemYedFJxgKVgd5KKY1ZwOMxrfV7AEqpBsBVsp5H7Q34A/u01nHpHosEximlSmutnwFigRjn+e8DxgB90k8lci6H6wXE6bSjivsDCUCXVMcqzPQ6Mri+ECIbJOkLYQNKqVuBupgk2RhorbXeqJRaCMRqrUcAI5zHbsSs4pVkKyZJO1KV+WISfOo1v72c5TVxLmajlPIDFLAd6AS8mcGGH08B/6e1TtqZ0UtrfdX5WE/gbeCKUirWWeaD+QKSoJT6K9V5fDALvrwOvJStN0YIkYYkfSHsYSAQrrU+6azNZyW5xq219kv/oFIqDJikta6cxXleBf6Triz5i0KqWNoqpRY4b68AHnTeXub8vUprfc75nGVAMecxdYFdWmuHcye3TphloIUQN0D69IXwcEqpAsAQTG09yQZnwh0I+Cul1iqlLiul/sUs4nG9rTpzYhJQAvDRWiugGvAP8BsmudcGvsJ0L3hhWg96p3p+ANAU+E0p1U4p9SlQFpgIPIlZ9a2jc5/wnUA55zmEEDdAavpCeL5BmH761FprrTcm3VFKzQDitdYJGZ1AKfUfIEZr/X5OLpx6UyDnEqEfOn8uYza1icJ8wdgPPK21npPu+VHASKXUHCAO6I5pKViD+ULQVWv9rXPcwUCt9Zc5iU8IkZbU9IXwYEqpisArwDuZPO6vlCqNqfH3V0qFOX/uSHfoPUCLdGVeSqliqX6KK6Uy2go42Nkk/zXwstb6KUzfv5/W+m/MUq0TgXeUUl8740kvGvgX05zfBjNOoB9wi1IqBLM3ezWZ9ifEzZGkL4Rn+wcYD+xKV57UvH8FqAhMA/pi+smnAzXSHZ9Iqn5+p4qYPdCTfs4D36c+QCnVyHntgkB9rfWbzodOApFKKW9tTMdszHIr6T53lFIdgW1AW0yrwApM68VnQHvMxjiVMbMQPlVKFcziPRFCZEKSvhAeTGudoLWelcFDrZ197AGYpPwt8LbW+kFMM/qObJz+qNZaJf1gPi+apjtmP1BHa/2A1vpwuscakWpGgNZ6LWZmQfLMAaXUOGAJMAA4gNmNrTBmsF4kZurh7UAIZgvcekCPbMQuhMiA9OkLYVPOpvCk5vC1mAFxe4ForfXxnJ7POY/+arri74CW12l1d2TymFJKeWFG7y/XWv/uLGyC2Q98HvCn1voJpVRD4KTW+h+lVD3nOb2vt32oECJjkvSFsKcNqW5XAT7HrIqngXddeJ17MYvoJCQtrqOUqgXsBk4AX2utn0w62DlPP2mKYF0gHIhVSsWnO28gJrmHpXoupKwV0AHY5MLXIUS+IElfCHtKWpzHB2dCdi6LOxgzJc4ltNZXUt937uu9FFgMvABsdibrcVrrK86V9OKcz40gk88gpdSXwF9a6ydcFasQQvr0hbALb1L+PScnUq11PFBIKTUB6AhEAPOVUmWcm+DUV0rVxkz5K6qUquWsqZcDfJLuO39udx5fLf3FlVIllFJPAnuAX4FRWuuTQDNM3/5+pdQopVTRXHsHhBBZkpq+EPbgT8qiNcmL1zg34FmFGREfghmFPwOTmOtiRs2nXnd/a7rzpr/v4zxfd+f5n8AsttMAM6hvWOq59M5++FbAMMxCPtOVUh9rrftl8XpSf4kRQriISrvHhRDCbpRSZbXWp9KVlUxa9vYmz30nZo7/l87m+usd64eZMviP1vqHLI5dAxzRWg+72RiFECkk6QshhBD5hDSfCSGEEPmEJH0hhBAin5CkL4QQQuQTkvSFEEKIfEKSvhBCCJFPSNIXQggh8on/B0EUxE1YQe6bAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 6))\n",
    "plt.plot(fpr, tpr, \"b:\", linewidth=2, label=\"SGD\")\n",
    "plot_roc_curve(fpr_forest, tpr_forest, \"Random Forest\")\n",
    "plt.legend(loc=\"lower right\", fontsize=16)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9919816325503626"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_train_5, y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9812692816218599"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred_forest = cross_val_predict(forest_clf, X_train, y_train_5, cv=3)\n",
    "precision_score(y_train_5, y_train_pred_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.821435159564656"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5, y_train_pred_forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 总结\n",
    "- 选择合适的指标利用交叉验证来对分类器进行评估\n",
    "- 选择满足需求的精度/召回率权衡\n",
    "- 使用ROC曲线和ROC AUC分数比较多个模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多类别分类器"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 尝试5之外的检测\n",
    "- 多类别分类器 区分两个以后的类别\n",
    "- 支持向量机SVM和线性分类器只可以处理二元分类器\n",
    "- 随机森林和朴素贝叶斯可以直接处理多个类别"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 解决方案\n",
    "1. 将数字图片分类0到9，训练10个二元分类器，每个数字一个，检测一张图片时，获取每个分类器的决策分数，哪个最高属于哪个，称为一对多OvA\n",
    "2. 为每一对数字训练一个二元分类器，区分0，1 区分0，2 区分1，2 称为一对一OvO策略，存在N个类别，需要N*（N-1）/2个分类器，最后看哪个类别获胜最多"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(random_state=42)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['5'], dtype='<U1')"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-18377.37264113, -26718.09807451, -11212.92758506,\n",
       "          2588.75734839, -22744.2300789 ,   2855.08991656,\n",
       "        -27115.54785488, -25401.43523884, -12935.79830601,\n",
       "         -8328.76210717]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "some_digit_scores = sgd_clf.decision_function([some_digit])\n",
    "some_digit_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(some_digit_scores)  # 最大值的索引"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'], dtype='<U1')"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.classes_  # 目标类别的分类标签，按值大小排列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'5'"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.classes_[np.argmax(some_digit_scores)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['3'], dtype=object)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用OvO策略，一对一或者一对多\n",
    "from sklearn.multiclass import OneVsOneClassifier\n",
    "ovo_clf = OneVsOneClassifier(SGDClassifier(max_iter=5, tol=-np.infty, random_state=42))\n",
    "ovo_clf.fit(X_train, y_train)\n",
    "ovo_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ovo_clf.estimators_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['5'], dtype=object)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用随机森林\n",
    "forest_clf.fit(X_train, y_train)\n",
    "forest_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., 0., 0., 1., 0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 随机森林直接将实例分为多个类别，调用predict_proba()可以获得分类器，将每个实例分为每个类别的概率列表\n",
    "forest_clf.predict_proba([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  评估分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.88925, 0.87075, 0.8761 ])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用交叉验证评估SGD的准确率\n",
    "cross_val_score(sgd_clf, X_train, y_train, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.8959 , 0.904  , 0.89985])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " # 将输入进行简单缩放 ，可以得到准确率 90 %以上\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train.astype(np.float64))\n",
    "cross_val_score(sgd_clf, X_train_scaled, y_train, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 错误分析"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 项目流程\n",
    "1、探索数据\n",
    "\n",
    "2、尝试多个模型\n",
    "\n",
    "3、选择最佳模型并进行网络搜索调参\n",
    "\n",
    "4、尽可能自动化\n",
    "\n",
    "\n",
    "###  确定一个相对合适的模型，进一步优化，分析其错误类型\n",
    "查看混淆矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5602,    0,   15,    8,    8,   34,   30,    4,  221,    1],\n",
       "       [   0, 6419,   40,   23,    3,   43,    4,    8,  193,    9],\n",
       "       [  29,   26, 5262,   84,   70,   23,   62,   33,  362,    7],\n",
       "       [  23,   18,  122, 5231,    0,  197,   28,   40,  407,   65],\n",
       "       [  13,   12,   45,    8, 5233,    9,   33,   21,  324,  144],\n",
       "       [  30,   19,   29,  155,   51, 4442,   73,   20,  538,   64],\n",
       "       [  25,   16,   56,    1,   46,   94, 5537,    6,  136,    1],\n",
       "       [  20,   12,   53,   22,   45,   13,    3, 5710,  182,  205],\n",
       "       [  15,   56,   38,  101,    2,  121,   31,    9, 5436,   42],\n",
       "       [  25,   19,   27,   60,  125,   37,    1,  164,  368, 5123]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred = cross_val_predict(sgd_clf, X_train_scaled, y_train, cv=3)\n",
    "conf_mx = confusion_matrix(y_train, y_train_pred)\n",
    "conf_mx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPkAAAEACAYAAABxpdD1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAKZElEQVR4nO3dTYid9RWA8efkS7TaaGyqSEQURaRIox1iLbWoKNiFiP3AgkSwLcEiuo60CHVhQcSNGGX86Eb64caiYBfSEqq0UqeYIkgEF34gBNtBTRUcjfd0kWmVOMl9J/P+7zv35PmtJpmb/xxu8sx7Z3LvmchMJNW1ZugBJLVl5FJxRi4VZ+RScUYuFWfkUnFG3kFEbIyIP0bEsxHxZERsGHqmLiLitIh4aeg5liMidkXEtUPP0UVEnBIRz0TEcxHx0NDzHM4gkUfEoxHx14j4xRAf/yjcCNyXmVcD+4BrBp6nq3uB44ceoquIuAw4PTOfHnqWjrYDj2fmZcBJETEz9EBLmXjkEfE9YG1mfgs4IyLOm/QMy5WZuzLz2cVfbgbeGXKeLiLiSuBDDn5SWvUiYj3wMPB6RFw39DwdzQPnR8TJwJnAm8OOs7QhruSXA08svv1n4NsDzHBUIuJS4JTMfGHoWY5k8cuJO4GdQ8+yDDcBrwD3ANsi4raB5+nieeA84HZgL/DusOMsbYjIvwS8vfj2fuC0AWZYtojYBNwP/HjoWTrYCTyQme8NPcgyXATMZuY+4HHgioHn6eJu4JbMvIuDkd888DxLGiLyD/js68QTB5phWRavjE8Ad2TmG0PP08FVwK0RsRvYGhGPDDxPF68B5yy+PQNMw/18AnBhRKwFLgFW5QtBYtIvUImIm4CvZua9EfFL4NXM/M1Eh1imiPgZBz9r/3Pxtx7MzN8POFJnEbE7My8feo5xIuIk4DEOPrJbD/wgM98+8p8aVkRsA34NnAX8Dbg+Mz8YdqovGiLyLwPPAX8Cvgt8MzPfn+gQ0jFk4pHDwf9fBK4G/rL4NZikRgaJXNLkrPpveklaGSOXihss8ojYMdTHPlrO3N60zQurf+Yhr+Sr+o45DGdub9rmhVU+sw/XpeJ6/e76pk2bcsuWLZ1uOz8/z6mnntrpti+//PJKxpKWLSI63zYzl337FjJzySHW9flBtmzZwlNPPdXnkQCcffbZvZ+pL1rOP9TVolUwxx13XJNzAT766KNmZy/Fh+tScUYuFWfkUnFGLhVn5FJxRi4V1ynyKdyuKmnR2MincbuqpM90uZJfzpRuV5XULfIjbleNiB0RMRcRc/Pz833PJ2mFukR+xO2qmTmbmTOZOdP1ueiSJqdL5P/gs4foXwdebzaNpN51eYHKH4DnIuIMFrerNp1IUq/GXskzcz8Hv/n2AnCF65Ol6dLppaaZ+S6ffYdd0hTxGW9ScUYuFWfkUnFGLhXX6yLHiGiycKvlj3Jas6bN57lp/PFTrXa8TeN90XLH28LCQpNzD7fI0Su5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFdfpZaMvRYq1vq7XJAHv27Gly7sUXX9zkXGi34ng0GjU5t+XfX6v7YhpXMh+OV3KpOCOXijNyqTgjl4ozcqk4I5eKM3KpOCOXihv7ZJiI2Aj8bvG2HwA3ZObHrQeT1I8uV/Ibgfsy82pgH3BN25Ek9WnslTwzd33ul5uBd9qNI6lvnZ+7HhGXAqdk5guH/P4OYEffg0nqR6fII2ITcD/w/UPfl5mzwOzi7dq8WkDSURv7NXlEbACeAO7IzDfajySpT12+8fYT4BvAzyNid0Tc0HgmST3q8o23B4EHJzCLpAZ8MoxUnJFLxRm5VJyRS8UZuVRc9LntchqfDLNuXe8LawGYm5trci7A1q1bm5zbakPpxx9P3+uZNm7c2Ozs/fv3937maDQiM5dcleyVXCrOyKXijFwqzsil4oxcKs7IpeKMXCrOyKXijFwqzsil4oxcKs7IpeKMXCrOyKXijFwqzsil4oxcKs7IpeKMXCrOyKXijFwqzsil4o75lcwRS26xXbE+79dD7dmzp8m5rVY9t7qPod39fOKJJzY5F2BhYaH3Mw8cOMBoNHIls3QsMnKpOCOXijNyqTgjl4ozcqk4I5eK6xR5RJwWES+1HkZS/7peye8Fjm85iKQ2xkYeEVcCHwL72o8jqW9HjDwiNgB3AjsnM46kvq0b8/6dwAOZ+d7hnn8cETuAHX0PJqkf4x6uXwXcGhG7ga0R8cihN8jM2cycycyZFgNKWpkjXskz8zv/ezsidmfmT9uPJKlPnf+fPDMvbziHpEZ8MoxUnJFLxRm5VJyRS8UZuVSckUvF9b6ttcVmzpabT1ttEl2/fn2Tc+HgZs4WnnzyySbnXn/99U3OBRiNRk3O3bx5c5NzAebn53s/czQakZlua5WORUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnG9b2vt7bAJWbOmzee5VltEod2G2Vb3xauvvtrkXIBzzz23ybktt+1+8sknTc51W6t0jDJyqTgjl4ozcqk4I5eKM3KpOCOXijNyqbjOkUfEroi4tuUwkvrXKfKIuAw4PTOfbjyPpJ6NjTwi1gMPA69HxHXtR5LUpy5X8puAV4B7gG0Rcdvn3xkROyJiLiLmWgwoaWW6RH4RMJuZ+4DHgSs+/87MnM3MmcycaTGgpJXpEvlrwDmLb88Ab7QbR1Lf1nW4zaPAYxHxI2A98IO2I0nq09jIM/M/wA8nMIukBnwyjFSckUvFGblUnJFLxRm5VJyRS8W5ktmVzP/X57+FSXnrrbeanNtq1TO0+ftbWFhgNBq5klk6Fhm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8VNxbbWVhtVod2G0rVr1zY5F+DAgQNNzl23rssPuV2+Tz/9tMm50O7vb+/evU3OBbjgggt6PzMzyUy3tUrHIiOXijNyqTgjl4ozcqk4I5eKM3KpuCNGHhGnRMQzEfFcRDw0qaEk9WfclXw78HhmXgacFBEzE5hJUo/GRT4PnB8RJwNnAm82n0hSr8ZF/jxwHnA7sBd4t/lEkno1LvK7gVsy8y4ORn7zoTeIiB0RMRcRcy0GlLQy4yI/AbgwItYClwBfeDVAZs5m5kxm+vW6tAqNi/xXwCzwPrAJ+G3ziST16oivLczMvwNfm9AskhrwyTBScUYuFWfkUnFGLhVn5FJxRi4VZ+RScW128PZsNBo1OztiyS22K9Zy5mlbndxqbTLAhg0bmpy7bdu2JucCvPjii72fuX379sO+zyu5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVRc9LlJMyL+BbzR8eZfAf7d2wefDGdub9rmhdUx81mZuXmpd/Qa+XJExFxmzgzywY+SM7c3bfPC6p/Zh+tScUYuFTdk5LMDfuyj5cztTdu8sMpnHuxrckmT4cN1qTgjl4ozcqk4I5eKM3KpuP8CAvxEYG6SRSUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 使用matplotlib的matshow 函数来查看混淆矩阵的图像表示\n",
    "plt.matshow(conf_mx, cmap = plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "大多数图片都在主对角线上，说明它们被正确分类\n",
    "\n",
    "数字5 看起来比较暗，说明：\n",
    "\n",
    "1、数字5图片较少  \n",
    "2、分类器在数字5上执行效果不如其他数字上好\n",
    "\n",
    "假设把焦点放在错误上，为取得错误率，而不是错误绝对值，需要将混淆矩阵中每个值除以相应类别中的图片数量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "row_sums = conf_mx.sum(axis=1, keepdims=True)\n",
    "norm_conf_mx = conf_mx / row_sums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPkAAAEACAYAAABxpdD1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAALKUlEQVR4nO3dX4jndbnA8fczvx2Z3VU3R801jIVlxYuICgdrQ0MjsS4kPHpICAUrliLq2iiCuigI6SbSmP4cL+RU3hQGBYaxHPMcOaxIILFKwhqGo7a4U9uyozPzdLEzKDo7v++s3898Zx7fr6vZnZ/PPAy+5zvzm+/vs5GZSKprYugFJLVl5FJxRi4VZ+RScUYuFWfkUnFG3kFE7ImI30XE7yPiVxFx3tA7dRERl0XEk0PvsRERcW9E3Dz0Hl1ExEUR8duIeDQifjT0PmczSOQR8dOI+N+I+MYQH/8cfBb4fmbeCMwBnxx4n67uAXYOvURXEXEdsDczfzP0Lh3dATyQmdcBF0TEzNALrWXTI4+I/wBGmflR4D0RceVm77BRmXlvZv5+5Y+XAi8NuU8XEfFx4F+c+aK05UXEJPBj4FhEfHrofTo6DlwVEe8C3gv8ddh11jbElfx64MGVt/8AXDvADuckIg4CF2Xm40Pvsp6VHye+Cdw99C4bcCfwZ+B7wDUR8ZWB9+nij8CVwFeBo8Arw66ztiEi3w38beXtfwCXDbDDhkXENPAD4HND79LB3cAPM/PE0ItswIeA2cycAx4Abhh4ny6+A3wxM7/NmcjvGnifNQ0R+Ule/znx/IF22JCVK+ODwNcy87mh9+ngE8CXI+Iw8MGI+MnA+3TxF2D/ytszwHb4PO8C3h8RI+DDwJZ8IUhs9gtUIuJO4N2ZeU9EfAt4OjP/e1OX2KCI+BJnvmr/aeWv7svMXw64UmcRcTgzrx96j3Ei4gLgZ5z5zm4SuC0z/7b+fzWsiLgG+C9gH/B/wC2ZeXLYrd5qiMgvBB4FHgE+BXwkM+c3dQnpHWTTI4czv18EbgT+Z+VnMEmNDBK5pM2z5Z/0kvT2GLlU3GCRR8ShoT72uXLn9rbbvrD1dx7ySr6lPzFn4c7tbbd9YYvv7LfrUnG9PrseEdvuqfqI6PzYzOz8+O34W4uJieG/5m/kcwywvLzcZI+LL76482NPnz7N1NRU58cfP378XFYaKzPX/MTtaPLRtpHJyckmc5eWlprMbWn37t1N5rb8gnfyZJsbzG6+ud1L2u+///5ms9cy/JduSU0ZuVSckUvFGblUnJFLxRm5VFynyLfh6aqSVoyNfDuerirpdV2u5NezTU9XldQt8nVPV42IQxFxJCKO9L2cpLevy22t656umpmzwCxsz3vXpeq6XMmf4PVv0T8AHGu2jaTedbmS/xp4NCLew8rpqk03ktSrsVfyzPwHZ558exy4weOTpe2l00tNM/MVXn+GXdI24h1vUnFGLhVn5FJxRi4V944/423nzp3jH3QOFhYWmsxtObvVQY7z8+1+IdNq5xdffLHJXIDRaNT7zPXOFPRKLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScb0eyTwxMcGuXbv6HAlAZrt/9rzVccEHDhxoMhfg1VdfbTL35ZdfbjL38ssvbzIX4OTJk03m3nrrrU3mAjz88MPNZq/FK7lUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxY29GSYi9gC/WHnsSeAzmdnmbgxJvetyJf8s8P3MvBGYAz7ZdiVJfRp7Jc/Me9/wx0uBl9qtI6lvne9dj4iDwEWZ+fib/v4QcGjl7X63k/S2dYo8IqaBHwBvuWs/M2eBWYDRaNTulSSSzsnYn8kj4jzgQeBrmflc+5Uk9anLE2+fB64Gvh4RhyPiM413ktSjLk+83Qfctwm7SGrAm2Gk4oxcKs7IpeKMXCrOyKXiej2tNTNZWlrqcyQAy8vLvc9cdfXVVzeZ+8QTTzSZ29Itt9zSZG7L00knJyebzN2zZ0+TuQB79+7tfeZLL539bnOv5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFdfrkcwAi4uLfY9s6tixY03mjkajJnOBJsdeAzz00ENN5u7evbvJXICFhYUmc59//vkmcwEOHDjQ+8z5+fmzvs8ruVSckUvFGblUnJFLxRm5VJyRS8UZuVRcp8gj4rKIeLL1MpL61/VKfg+ws+UiktoYG3lEfBz4FzDXfh1JfVs38og4D/gmcPfmrCOpb+PuXb8b+GFmnoiINR8QEYeAQ30vJqkf475d/wTw5Yg4DHwwIn7y5gdk5mxmzmTmzNm+EEgazrpX8sz82OrbEXE4M7/QfiVJfer8e/LMvL7hHpIa8WYYqTgjl4ozcqk4I5eKM3KpOCOXiovM7G3YaDTK888/v7d5q1577bXeZ67asaP3A2sBuPbaa5vMBXjyyTYvCJyba/PyhP379zeZC/DCCy80mXvq1KkmcwH27dvX+8y5uTkWFhbWvBvNK7lUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VFzvp7VOTU31Nm/V4uJi7zNXTU9PN5nb6uRTgEsuuWRbzT169GiTuQATE22uUwcPHmwyF+Cxxx5rMjczPa1Veicycqk4I5eKM3KpOCOXijNyqTgjl4ozcqm4zpFHxL0RcXPLZST1r1PkEXEdsDczf9N4H0k9Gxt5REwCPwaORcSn268kqU9druR3An8GvgdcExFfeeM7I+JQRByJiCN93gcvqR9dIv8QMJuZc8ADwA1vfGdmzmbmTGbORKx5f7ykAXWJ/C/A/pW3Z4Dn2q0jqW87Ojzmp8DPIuJ2YBK4re1Kkvo0NvLM/Cfwn5uwi6QGvBlGKs7IpeKMXCrOyKXijFwqzsil4rr8nryzzGR5ebnPkQBNZq5aWlpqMnfnzp1N5kK7z8ezzz7bZG7LOyFbfS4uvPDCJnMBbrut/1tNHnnkkbO+zyu5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVRcZGZvwyYmJnJycrK3eaump6d7n7nq9OnTTeZeccUVTeYCPPXUU03mzszMNJn79NNPN5kLcOrUqSZzW53iC23+f56fn2dxcXHNY3G9kkvFGblUnJFLxRm5VJyRS8UZuVSckUvFrRt5RFwUEb+NiEcj4kebtZSk/oy7kt8BPJCZ1wEXRESbuyUkNTMu8uPAVRHxLuC9wF+bbySpV+Mi/yNwJfBV4CjwSvONJPVqXOTfAb6Ymd/mTOR3vfkBEXEoIo5ExJE+74OX1I9xke8C3h8RI+DDwFsqzszZzJzJzJmINe+PlzSgcZF/F5gF5oFp4OfNN5LUqx3rvTMz/x943ybtIqkBb4aRijNyqTgjl4ozcqk4I5eKM3KpOCOXiuv1SObRaJRTU1O9zVu1vLzc+8xVo9Goydw9e/Y0mQvtjnt+5plnmsw9ceJEk7kAN910U5O5t99+e5O5AHfd9Za7w3uRmR7JLL0TGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxfV6WmtEvAw81/HhlwB/7+2Dbw53bm+77QtbY+d9mXnpWu/oNfKNiIgjmTkzyAc/R+7c3nbbF7b+zn67LhVn5FJxQ0Y+O+DHPlfu3N522xe2+M6D/UwuaXP47bpUnJFLxRm5VJyRS8UZuVTcvwFjjlJT0XTCWAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 用0填充对角线 只保留错误，重新绘制\n",
    "np.fill_diagonal(norm_conf_mx, 0)\n",
    "plt.matshow(norm_conf_mx, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "行代表实际类别，列代表预测类别\n",
    "\n",
    "8列比较亮，说明许多图片被错误的分类为数字8\n",
    "\n",
    "9行也偏亮，说明数字9经常会跟其他数字混淆\n",
    "\n",
    "有些很暗，比如行1，大多数数字1都被正确的分类，但有一些和8混淆\n",
    "\n",
    "最亮的是（5，8），说明5和8是错误最多的\n",
    "\n",
    "因此，\n",
    "\n",
    "- 改进数字8的分类\n",
    "- 修正数字5和8的混淆"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 优化分类器\n",
    "- 尝试多收集训练集\n",
    "- 开发新特征\n",
    "- 优化算法\n",
    "- 对图片进行预处理\n",
    "- 分析单个错误"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-61-30cc1215e802>:19: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  X_aa = X_train[(y_train == cl_a) & (y_train_pred == cl_a)]\n",
      "<ipython-input-61-30cc1215e802>:20: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  X_ab = X_train[(y_train == cl_a) & (y_train_pred == cl_b)]\n",
      "<ipython-input-61-30cc1215e802>:21: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  X_ba = X_train[(y_train == cl_b) & (y_train_pred == cl_a)]\n",
      "<ipython-input-61-30cc1215e802>:22: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  X_bb = X_train[(y_train == cl_b) & (y_train_pred == cl_b)]\n"
     ]
    },
    {
     "ename": "ZeroDivisionError",
     "evalue": "integer division or modulo by zero",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mZeroDivisionError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-61-30cc1215e802>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m8\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m8\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msubplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m221\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m;\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 26\u001b[1;33m \u001b[0mplot_digits\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_aa\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m25\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mimages_per_row\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     27\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msubplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m222\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m;\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m \u001b[0mplot_digits\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_ab\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m25\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mimages_per_row\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-61-30cc1215e802>\u001b[0m in \u001b[0;36mplot_digits\u001b[1;34m(instances, images_per_row, **options)\u001b[0m\n\u001b[0;32m      3\u001b[0m     \u001b[0mimages_per_row\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minstances\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mimages_per_row\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mimages\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0minstance\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0minstance\u001b[0m \u001b[1;32min\u001b[0m \u001b[0minstances\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m     \u001b[0mn_rows\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minstances\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m//\u001b[0m \u001b[0mimages_per_row\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m     \u001b[0mrow_images\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m     \u001b[0mn_empty\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mn_rows\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mimages_per_row\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minstances\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mZeroDivisionError\u001b[0m: integer division or modulo by zero"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPYAAADmCAYAAADmxHhTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAK/ElEQVR4nO3db6jdB33H8fdnSQsx6WrKLpEMO5CFFodE8VJnSMdtacCCInSbLUgF/xAcrj6uLPhAgoMgheGscjWDQmHQPbBDqYxCFozLuu0GCaxOmA8SISMYJXoXH+xB+e7BPXrj4d57fvf03NPbr+8XhP5u7/ec870nfd/fuSe/klQVknr5nTd6AUmzZ9hSQ4YtNWTYUkOGLTVk2FJDhi01NCjsJIeSXNji83ck+XaSi0k+Mbv1JE1jYthJDgLPAfu3GHsKWKmqY8AHk9w1o/0kTWHvgJnXgMeBf9xiZgl4enR8EVgE/vn2gSQngZMA+/fvf+/999+/3V2l3zqXLl36aVUtbPd2E8OuqlWAJFuN7QeujY5XgUMb3M8ysAywuLhYKysr291V+q2T5Oo0t5vVm2e3gH2j4wMzvF9JU5hVgJeA46Pjo8CVGd2vpCkM+Rn7NyR5GHhnVf3tbf/6OeClJA8C7wT+bUb7SZrC4DN2VS2N/nluLGqq6ipwAvgX4JGqem2WS0ranm2fsTdTVf8DvDCr+5M0Pd/kkhoybKkhw5YaMmypIcOWGjJsqSHDlhoybKkhw5YaMmypIcOWGjJsqSHDlhoybKkhw5YaMmypIcOWGjJsqSHDlhoybKkhw5YaMmypIcOWGjJsqSHDlhoybKkhw5YaMmypIcOWGjJsqaFBYSc5m+RiklObfP5gkpeSXEjytdmuKGm7Joad5DFgT1UdAw4nObLB2JPA81X1IHBXksUZ7ylpG4acsZdY/wvtzwHHN5j5GXBfkrcCbwd+PIvlJE1nSNj7gWuj41Xg0AYz3wOOAJ8FfgjcHB9IcjLJSpKVGzduTLmupCGGhH0L2Dc6PrDJbb4IfLqqvsBa2B8fH6iq5aparKrFhYWFafeVNMCQsC+x/vL7KHBlg5m3AO9Ksgd4H1Az2U7SVIaE/SLwZJJngI8AryY5PTbz18Ay8AvgHuDvZ7mkpO3ZO2mgqlaTLAEngDNVdR24PDbz78Af7cSCkrZvYtgAVXWT9XfGJe1yXnkmNWTYUkOGLTVk2FJDhi01ZNhSQ4YtNWTYUkOGLTVk2FJDhi01ZNhSQ4YtNWTYUkOGLTVk2FJDhi01ZNhSQ4YtNWTYUkOGLTVk2FJDhi01ZNhSQ4YtNWTYUkOGLTVk2FJDhi01ZNhSQ4YtNTQo7CRnk1xMcmrC3LNJPjSb1SRNa2LYSR4D9lTVMeBwkiObzD0IvK2qvjXjHSVt05Az9hLwwuj4HHB8fCDJHcDXgStJPrzRnSQ5mWQlycqNGzemXFfSEEPC3g9cGx2vAoc2mPkY8APgDPBAkqfGB6pquaoWq2pxYWFh2n0lDTAk7FvAvtHxgU1u8x5guaquA88DD81mPUnTGBL2JdZffh8Frmww8yPgHaPjReDq695M0tT2Dph5EbiQ5DDwKPBEktNVdfs75GeBv0vyBHAH8Gcz31TSYBPDrqrVJEvACeDM6OX25bGZ/wX+fCcWlLR9Q87YVNVN1t8Zl7TLeeWZ1JBhSw0ZttSQYUsNGbbUkGFLDRm21JBhSw0ZttSQYUsNGbbUkGFLDRm21JBhSw0ZttSQYUsNGbbUkGFLDRm21JBhSw0ZttSQYUsNGbbUkGFLDRm21JBhSw0ZttSQYUsNGbbUkGFLDQ0KO8nZJBeTnJowdyjJ92ezmqRpTQw7yWPAnqo6BhxOcmSL8S8B+2a1nKTpDDljL7H+l96fA45vNJTkYeCXwPWZbCZpakPC3g9cGx2vAofGB5LcCXweeHqzO0lyMslKkpUbN25Ms6ukgYaEfYv1l9cHNrnN08BXqurnm91JVS1X1WJVLS4sLGx7UUnDDQn7Eusvv48CVzaYeQT4TJLzwLuTfGMm20mayt4BMy8CF5IcBh4Fnkhyuqp+/Q55Vf3Jr46TnK+qT818U0mDTQy7qlaTLAEngDNVdR24vMX80qyWkzSdIWdsquom6++MS9rlvPJMasiwpYYMW2rIsKWGDFtqyLClhgxbasiwpYYMW2rIsKWGDFtqyLClhgxbasiwpYYMW2rIsKWGDFtqyLClhgxbasiwpYYMW2rIsKWGDFtqyLClhgxbasiwpYYMW2rIsKWGDFtqyLClhgxbamhQ2EnOJrmY5NQmn787yXeSvJzkm0nunO2akrZjYthJHgP2VNUx4HCSIxuMfRR4pqpOANeBD8x2TUnbsXfAzBLwwuj4HHAc+O/bB6rq2ds+XAB+Mn4nSU4CJwHuvffeKVaVNNSQl+L7gWuj41Xg0GaDSd4PHKyqV8Y/V1XLVbVYVYsLCwtTLStpmCFn7FvAvtHxATb5ZpDkHuDLwJ/OZjVJ0xpyxr7E2stvgKPAlfGB0ZtlLwCfq6qrM9tO0lSGhP0i8GSSZ4CPAK8mOT0280ngvcBfJTmf5PHZrilpOya+FK+q1SRLwAngTFVdBy6PzXwV+OpOLChp+4b8jE1V3WT9nXFJu5xXnkkNGbbUkGFLDRm21JBhSw0ZttSQYUsNGbbUkGFLDRm21JBhSw0ZttSQYUsNGbbUkGFLDRm21JBhSw0ZttSQYUsNGbbUkGFLDRm21JBhSw0ZttSQYUsNGbbUkGFLDRm21JBhSw0ZttTQoLCTnE1yMcmp1zMjaT4mhp3kMWBPVR0DDic5Ms2MpPkZcsZeYv0vvT8HHJ9yRtKc7B0wsx+4NjpeBf5wmpkkJ4GTow//L8l/bm/Vufs94Kdv9BJb2O37gTvOwn3T3GhI2LeAfaPjA2x8lp84U1XLwDJAkpWqWtz2tnO023fc7fuBO85CkpVpbjfkpfgl1l9aHwWuTDkjaU6GnLFfBC4kOQw8CjyR5HRVndpi5o9nvaik4SaesatqlbU3x14BHqqqy2NRbzTziwl3uzzVtvO123fc7fuBO87CVPulqma9iKQ3mFeeSQ0ZttTQjob9ZrgUddLjJ7k7yXeSvJzkm0nu3E373TZ3KMn357XX2GMP3fHZJB+a1163Pe6k3+ODSV5KciHJ1+a93217HEpyYYvP35Hk26Ov5RNb3deOhf1muBR14ON/FHimqk4A14EP7LL9fuVLrF9LMDdDd0zyIPC2qvrWLtzvSeD5qnoQuCvJ3P9cO8lB4DnWLvbazFPAyuhr+WCSuzYb3Mkz9hK7/1LUiY9fVc9W1cujDxeAn8xnNWDg85PkYeCXrH3jmbclJuyY5A7g68CVJB+e32rAsOfwZ8B9Sd4KvB348Vw2+02vAY+zduXmZpZY/1ouApt+A9rJsMcvMz005cxOGvz4Sd4PHKyqV+ax2MjE/UY/GnweeHqOe91uyHP4MeAHwBnggSRPzWk3GLbf94AjwGeBHwI357PauqpaHfDHxIP/e93JsGdyKeoOG/T4Se4Bvgxs+XPNDhiy39PAV6rq5/NaasyQHd8DLFfVdeB54KE57QbD9vsi8Omq+gJrYX98Trtt1+BedjKkN8OlqBMff3RGfAH4XFVdnd9qwLDn5xHgM0nOA+9O8o35rPZrQ3b8EfCO0fEiMM/ncch+bwHelWQP8D5gt17cMbyXqtqRX8DvApeBZ4D/Gi1yesLM3Tu1z+vY8S9Ye2l2fvTr8d2039j8+Xk+f9t4Du8C/gH4LvCvwO/vsv0eAF5l7Yz4MnBg3s/j+O8h8DDwl2Of+4PRnn8D/AdrbwpueD87euXZ6J2+E8B3a+1l2FQzO+mNfvxJdvt+sPt33O37bcfo/8c4DvxTbfEzuZeUSg155ZnUkGFLDRm21JBhSw0ZttTQ/wNlIOpJq7EJygAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_digits(instances, images_per_row=10, **options):\n",
    "    size = 28\n",
    "    images_per_row = min(len(instances), images_per_row)\n",
    "    images = [instance.reshape(size,size) for instance in instances]\n",
    "    n_rows = (len(instances) - 1) // images_per_row + 1\n",
    "    row_images = []\n",
    "    n_empty = n_rows * images_per_row - len(instances)\n",
    "    images.append(np.zeros((size, size * n_empty)))\n",
    "    for row in range(n_rows):\n",
    "        rimages = images[row * images_per_row : (row + 1) * images_per_row]\n",
    "        row_images.append(np.concatenate(rimages, axis=1))\n",
    "    image = np.concatenate(row_images, axis=0)\n",
    "    plt.imshow(image, cmap = matplotlib.cm.binary, **options)\n",
    "    plt.axis(\"off\")\n",
    "\n",
    "\n",
    "# 查看数字3和数字5的例子\n",
    "cl_a, cl_b = 3, 5\n",
    "X_aa = X_train[(y_train == cl_a) & (y_train_pred == cl_a)]\n",
    "X_ab = X_train[(y_train == cl_a) & (y_train_pred == cl_b)]\n",
    "X_ba = X_train[(y_train == cl_b) & (y_train_pred == cl_a)]\n",
    "X_bb = X_train[(y_train == cl_b) & (y_train_pred == cl_b)]\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "plt.subplot(221); \n",
    "plot_digits(X_aa[:25], images_per_row=5)\n",
    "plt.subplot(222); \n",
    "plot_digits(X_ab[:25], images_per_row=5)\n",
    "plt.subplot(223);\n",
    "plot_digits(X_ba[:25], images_per_row=5)\n",
    "plt.subplot(224); \n",
    "plot_digits(X_bb[:25], images_per_row=5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "原因：SGD是一个线性模型，它所做就是为每个像素分配一个各个类别的权重，当它看到新的图像，将加权后的像素强度汇总，从而得到一个分数进行分类\n",
    "\n",
    "数字3和5在一部分像素位上有区别，所以分类器很容易将其弄混\n",
    "\n",
    "通过上面图像，如果书写3 的连接点左移，分类器可能将其分类为数字5，这个分类器对图像位移和旋转敏感\n",
    "\n",
    "减少混淆的方法之一，就是对图像进行预处理，确保位于中心位置并且没有旋转"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  多标签分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False,  True]])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "y_train = y_train.astype(np.int)\n",
    "y_train_large = (y_train >= 7)\n",
    "y_train_odd = (y_train % 2 == 1)\n",
    "y_multilabel = np.c_[y_train_large, y_train_odd]\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "knn_clf.fit(X_train, y_multilabel)\n",
    " \n",
    "# knn支持多标签分类，不是所有的分类器都支持\n",
    "knn_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "评估多标签分类器方法很多，方法之一就是测量每个标签的F1分数，或者其他二元分类器指标，然后简单平均"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_knn_pred = cross_val_predict(knn_clf, X_train, y_multilabel, cv=3)\n",
    "f1_score(y_multilabel, y_train_knn_pred, average=\"macro\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 多输出分类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "构建一个系统：去除图片中的噪声，分类器输出是多个标签值（0-255），像素值。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 增加噪声\n",
    "noise = np.random.randint(0,100,(len(X_train), 784))\n",
    "X_train_mod = X_train + noise\n",
    "\n",
    "noise = np.random.randint(0, 100, (len(X_test), 784))\n",
    "X_test_mod = X_test + noise\n",
    "\n",
    "y_train_mod = X_train\n",
    "y_test_mod = X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAC2CAYAAAD5uGd5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaDUlEQVR4nO3de4zU1dkH8O+zyC4IiFwU5CI3EQHLWhgvIAo2omljk6JV+laNqa20aWJtmjSNhZrWS038g5o00VcEG6X6NqaNtjFWawsWRKEsiIJc5V4EEQQWEJfLPu8f7Kq753tgZnZmmDN+P0lTeHxm5zczh8OPOec5j7k7REQkHVWn+wJERCQ3mrhFRBKjiVtEJDGauEVEEqOJW0QkMZq4RUQSo4lbRCQxZ+T7QDObDWA4gJfd/cFYXteuXb13795Z/Uy2p7yqiv/d0tjYGMTOPPNMmtvQ0JDVc8WYGY3X1NRk/XM//fTTrJ4r9vh27doFserqapp75MiRIBZ7b1hu7OcePnw4iHXo0IHmstcbex+PHz8exDp27Ehz2fWuWrVqt7ufQx+Qo2zHNQD07NnTBw4cWIinFQls3rwZu3fvpn9o8pq4zexGAO3cfZyZPWZmQ919Pcvt3bs3nnjiiRYx9gcVAI4ePRrEYhPOJ598EsQymQzN3bBhQ1bPFXPGGfxtGjp0aNY/d82aNUGM/eXDYgDQuXPnIDZo0CCau2nTpiB2ySWX0Nxt27YFsfPPP5/mrly5MohdeOGFNHfdunVBLPY+7t+/P4iNGjWK5m7dujWI1dbWbqHJOcplXAPAwIEDUVdXV4inFgnE5jMg/69KJgJ4vunXcwGM/+J/NLOpZlZnZnXsD6VImZqIk4xroOXY/uijj0p5bSKfyXfi7gRge9Ov6wH0+uJ/dPeZ7p5x90zXrl3bcn0ipXTScQ20HNvnnFOQb2dEcpbvd9wHATR/CdkZJ/kLoLq6Gv37928Ri03m7OuAhQsX0txOnToFseXLl9PcK6+8Moixr08AYNiwYUHsvffeo7nsu/PYVzvse372Vcf69fxf5r16BXMI/c45dg2xu8O+ffsGsfnz59PcHj16BLH6+vqsr2HHjh00l+nSpQuNs6/ICijrcS1yOuU7MJfi839G1gLYXJCrETm9NK4lCfnecb8IYIGZ9QHwdQBXFOyKRE6fF6FxLQnI647b3etxYiFnEYBr3F0rkJI8jWtJRd77uN19Lz5fgRepCBrXkgItvoiIJCbvO+5sNTY24uDBgy1isSIVtvOB7XoA+C6N2I4DtvMhl61cu3fvpnFWqLJ48WKay3ZZnHfeeUGs9Q6cZhs3bgxie/fupbmsKCb2GrZv3x7EBg8eTHNZNSOr6AR49SXb3QPwCtTY+3juuefSuMiXie64RUQSo4lbRCQxmrhFRBKjiVtEJDFFX5ysqanBkCFDWsTY6XUAsGfPniAWK49ftmxZEOvevTvNZQucscOvWBl77DhSdrLeZZddRnPfeeedIMYW9mKnA7Lr6tmzJ81lJfqTJk2iuWyh+N1336W5Bw4cCGKx95wtNMdOHdy3b18QGzFiBM1dtGgRjYt8meiOW0QkMZq4RUQSo4lbRCQxmrhFRBKjiVtEJDFF31UChLsnhg8fTvOWLFkSxGLNAlgPxViDWdbrcNWqVTSXNb+N9Ws866yzgtjatWtp7ujRo4MYa5pwwQUX0McfO3YsiMV2oAwYMCCIHTp0iOaynS1s9wjAjwmI7c6ZOHFiEGvfvj3N/fDDD4NYrGGxus6I6I5bRCQ5mrhFRBKjiVtEJDGauEVEEpPz4qSZnQFgY9P/AOBud19xkvygZPyDDz6guWxB6uOPP6a57GewRUiALzjGzpxm52Gz0naAl7HHzotufSY5wDu3x84qZwuOu3btorms3Dx2VvmMGTOCWKyUnp2nfe2119Jcdw9isVJ69p5XVfF7ith55YWQ69iW8nbppZcGMXYuPsD/HIwZM6bg11Qo+ewqGQXg/9z9F4W+GJHTTGNbkpDPVyVXAJhsZm+Y2bNNdykilUBjW5KQz8S9BMAEdx8PYB+AbxT0ikROH41tSUI+dxTvunvzGaNrAAxtnWBmUwFMBYr7naRIgeU0tmPH1IoUWz533HPMrNbM2gGYDCBYoXP3me6ecfeMKt0kIRrbkoR87rjvB/AcAAPwN3f/58mSGxoagg7l7OB8gJe3X3311Vlf2NKlS2l85MiRQSy2s4Ud1H/kyBGay0rhWRk8APzrX/8KYqxJROwujpWmjxo1iubOnTs3iMWOGaivrw9iq1evprnsPWNNKmJiO0VYKTzblQIAK1YUdZNHTmNbygPbRQUAu3fvDmJbtmyhubNmzQpiFbWrxN1X4sTqu0hF0diWVKgAR0QkMZq4RUQSo4lbRCQxRS8wqKqqQufOnVvEYmXhrDR9w4YNNHfnzp1BjJWFA7xkPdadnC2UvfjiizT329/+dhD7wQ9+QHPvueeeINatW7cg9tZbb9HHs4XMWPnu2LFjg9i8efNoLitDj53HPWXKlCAWO2aAnSsee8/ZovTChQtpbk1NDY1Lfh588EEa/9WvfhXEpk+fTnMfeOCBgl5TrmKbEthCZGzROzW64xYRSYwmbhGRxGjiFhFJjCZuEZHEaOIWEUlM0XeVmFmwE6BHjx40d+/evUEsVs7Kdn/EdpWwZgyxxgJ79uwJYrEGAOx1vPHGGzSX7TZhr7dPnz708a135gDA8uXLaS57vd/73vdo7rBhw4LY2WefTXOvv/76ILZjxw6ayxpCrF27luZu3749iMU62LMdM5Id9mdp9uzZNLd18xMA+O1vf0tzS7mrhB3HcMcdd9Bc9hpiR2jcddddbbuwEtMdt4hIYjRxi4gkRhO3iEhiNHGLiCSm6IuTjY2NwUJgrPyaHUxfW1tLc9m5zMuWLaO5bNEyVnb/k5/8JIi99NJLNJctdMTOyGbXcOzYsSD2+uuv08ez9ya2MHjfffcFsYEDB9JcthD5rW99i+ayBVLW+R0ANm/eHMR69+5Nc/fv359VDABWrlxJ43JqrIw9dj41Kw3P5Wz8topd18SJE4PYrl27aC5bnPzud79Lc0ePHp39xZUB3XGLiCRGE7eISGI0cYuIJEYTt4hIYjRxi4gkJqtdJWbWC8Cf3f0qM2sP4AUA3QHMcvenTvbYqqqq4MB/1jAB4Dsf2O4EgB/gH+s4zg7fb915vhkrtc5kMjS3oaEhiLFVb4CX2Hfq1CmITZgwgT5+yZIlQSzWdIE1MYi9XtbcILYDZdCgQUHs+PHjNHfo0KFZ57LjC0rVMKEtYzs1L7zwQhBjOy8AvotpxowZBb+mmCeffJLGWef22GsYMWJEELvxxhvbdmFl4pR33GbWDcDTAJpnmbsB1Ln7OAA3mFmXIl6fSNFobEuqsvmq5DiAKQDqm34/EcDzTb9+EwC/HRUpfxrbkqRTTtzuXu/uX6yG6ASg+Ti3egC9Wj/GzKaaWZ2Z1bGT6kTKQVvHduzkSpFiy2dx8iCAjk2/7sx+hrvPdPeMu2diDWJFylBOY5t9DyxSCvmUvC8FMB7AnwHUAlh0smQzCxYPYot9rJw5l1LUWLdnttD1yCOP0Fz2L4S3336b5v7sZz8LYmxhEADq6uqCGFsIZZ3fAWDfvn1B7D//+Q/NPXLkSBBbv349zX3uueeCWGyRd+vWrUGMnV8OAF/96leDGFvMBXgH+y5d+NfL/fr1o/ECyWlsl6vbb7+dxllpeGxh79Zbbw1ixSoLnz9/fhB76KGHaC673ljn9meeeSaI9ezZM8erK0/5TNxPA3jZzK4CMALA4sJekshpo7EtScj6qxJ3n9j0/1sATAKwEMC17s73eIkkQmNbUpPX6YDu/gE+X30XqRga25ICVU6KiCRGE7eISGJK0kih9W4CVrYK8NLnWLdvtpuBdRYHeDd2tvMC4CX2V111Fc295557gljsUHe2LZI1Jog1G5g2bVoQi+3o2LZtWxCbPn06zW19HAEQbzLBdrGwsmKA7/CJ7ekfMmRIEIt9PvX19TQun1uzZg2Nsx0Zsc/vl7/8ZUGv6WQefvjhIBbb7cLikydPprkXXXRR2y6sjOmOW0QkMZq4RUQSo4lbRCQxmrhFRBJTksXJQ4cOtYjFzuPu379/EFu7di3NZedbV1dX01y2qLZoEa9mZiXrN9xwA81l18YWWAHe/Z0t1s2ePZs+vvV7CAAdO3YkmbzcPHYO8YEDB4LYa6+9RnOHDRsWxFgpP8DfB9ZRHgA++OCDIBZbpI2VN39ZsUXg2NEP7L2LLewVozQ8tkD+6quvBrHY58zOh/nLX/7StgtLkO64RUQSo4lbRCQxmrhFRBKjiVtEJDFFX5zs0KFDUMG0Y8cOfjGkAXAsl4lVTu7fv5/GmQULFgSx2ILjs88+G8TGjRtHc994440gxs75fv311+njWcUYa94LAO+//34Q+/nPf05zWcNidu42wF/bgAEDaO7evXuDWKwClTVY3rRpE82NVVR+Wa1evTqIxaoOGdZAGABeeeWVrH8G+1zZdbFFSCC362W5l156adaPjxk+fHgQGz9+fNaPjy3mFqs5se64RUQSo4lbRCQxmrhFRBKjiVtEJDGauEVEEpPVrhIz6wXgz+5+lZn1xYkmqs1bF252949ijz106FBQXj5y5Eiay0rIv/a1r9FctoMkdlbzxo0bgxgrrwd4N3VWkg3w7vErVqyguX/961+DGNshEevyzlbTY8cBsJ0eO3fupLns7G5WMg8As2bNCmJXX301zWVl80899RTN/fWvfx3ELrvsMpob+yzy1ZaxXQ7YbohYuTiLr1q1Kuvc2O4PduxBLt3Ys31+gJ93/+GHH9LcXK6BHRMwZ86cNv/c3/3ud0Hspz/9Kc3NxSknbjPrhhPdr5v3jV0O4CF3f7zNzy5yGmlsS6qy+arkOIApAJpvZ68A8GMze8vMwr9ORNKhsS1JOuXE7e717v7FCpa/Axjn7mMBXGhmwbF3ZjbVzOrMrI599SBSDto6tj/6qKy/RZEKls/i5Jvu3nwW6BoAQ1snuPtMd8+4eyZ2lKdIGcppbLMjRkVKIZ+S91fN7H8A7AdwPYCZJ0uuqanBhRde2CIWW+QYM2ZMEIstnrAmt7GFq9tuuy2ILVmyhOayBc5YmXXrJsgA0K5dO5rLFjKHDg3mBSxfvpw+njUWPv/882nuddddF8QeeeQRmsvKldl1AcDChQuD2Pz582luJpMJYrGy4PXr1wexH/3oRzQ39rkVSE5juxywxUnWWBrg51bHFriZtpamFyL3pptuCmKx87jZz401EC6H9yEX+UzcvwEwD8ARAP/r7tm/YpHyprEtSch64nb3iU3/Pw9A5fa9ly8djW1JjQpwREQSo4lbRCQxmrhFRBJjxe6aPXz4cH/mmWeyymW7NFh3c4B3OG+9e6UZK+vu0aMHzWWrwLFmAbFu2szll18exFgn86oq/nfpXXfdFcRuueUWmst2sMSaLhw7diyIzZ07l+aynSJnnXUWzf33v/8dxGK7YFgDjXXr1tFctitg0KBBS909vLgiy2QyHutyX4li+9bPPffcIJZLWTjr/v7AAw/keHWVJ5PJoK6ujm5L0R23iEhiNHGLiCRGE7eISGI0cYuIJKboXd6rq6vRp0+fFrFYCTlbQPv0009p7scffxzEYuWl3bt3D2Kx8ni2+NX6PPFmrOv5zTffTHNZyXqvXr2CGFvEjD1Xv379aC472GvevHk0ly0MssVNgC/Gxs5Lr62tDWKstB3gJfbXXHMNzdWhZafPww8/TOPszx2LTZ48mT7+3nvvbduFfQnpjltEJDGauEVEEqOJW0QkMZq4RUQSo4lbRCQxRd9VcuzYsWAHyMUXX0xzWbdmthsD4CXrsRJ0VtY9alTQlQoAb4QQ29ny/PPPBzHW3RwADhw4EMQ++eSTIPbDH/6QPp4dmH/48GGay17D4MGDae7mzZuD2MCBA2kua17BmisAQP/+/YNYrOSdHWLPdtEAwP79+2lcCuuVV14JYo8++ijNZaXsrDtQrOGB5E533CIiidHELSKSGE3cIiKJ0cQtIpKYUy5OmllXAH9qyj0IYAqAxwEMB/Cyuz94sscfPXo0WHQ8fvx41hcYW9B68803gxhbwAP4edzt27enuayUPlaGPmnSpCB29tln09zzzjsviD3xxBNBrEOHDvTx//3vf4PY0aNHaS5b0I3ljhw5MoixhSUA2LVrVxDr27cvzWUd3VeuXJl17p49e2huobR1XFc6Vt6eS8fyOXPmFPJypJVs7rhvBTDD3ScB2AngOwDaufs4AH3MLDxoQqT8aVxLsk55x+3uj33ht+cAuA3Ao02/nwtgPAB+epBImdK4lpRl/R23mY0F0A3ANgDbm8L1AIIj7sxsqpnVmVmd9t1KOctlXDflfza2Y628RIotq4nbzLoD+D2AO3Hi+8Dmho+d2c9w95nunnH3TNeuXQt1rSIFleu4BlqO7dhagEixnXLiNrNqAM8DuNfdtwBYihP/jASAWgCbi3Z1IkWicS0py6bk/fsAxgCYZmbTAPwBwO1m1gfA1wFccbIH19TUBF3St2zZQnNHjBgRxGI7ULp06RLEWEk2wEu46+vrae7OnTuD2B//+Eeau2HDhiAWK9GfOnVqEGM7PWJl7EOGDAlisVV+VoZ+ySWX0Nw1a9YEMVYyD/AGGK0/22bz588PYrGO8OzzOXToEM1lxxfkqU3julLMnDmTxtnnFxtv999/fxC77rrr2nZhclLZLE4+jhPbpD5jZn8DMAnAI+6uL7ElORrXkrK8Dply97048c9MkYqhcS2pUOWkiEhiNHGLiCSm6OdxA+GiBussDvAy59heWbYQycrCYz9jwoQJNJedTx3res7K5mOl2rfccksQ27RpUxCLLb7t3r07iLHu9QDQqVOnINbY2Ehz2QLpkiVLaG5VVfj3PDtDHeDnccfs2LEjiLEjAoD42eiSH7Y4DWTfuR0Apk+fXtBrklPTHbeISGI0cYuIJEYTt4hIYjRxi4gkRhO3iEhiir6rpLq6Gv369WsR69WLHryGRYsWBTHWzR3guwtindtZN3XWiAHgzRjuu+8+mtvQ0BDEYqchsnJv9hpiBxfFSsAZ9nOXLVtGc3v37h3EamtraS7bmcJ2uwBAt27dghjbPQLw5hGrV6+muazxg+RvwYIFNM46t0v50B23iEhiNHGLiCRGE7eISGI0cYuIJKboi5MNDQ1BGXmsbJmVSW/fvp1k8pLoWLk4K2OPdYRn53Gz86IBvmD4zW9+k+ayDuls4XXw4MH08axs/+DBgzR39OjRQWzdunU0l51rnkvZ/d69e2kuex/79OlDczt27BjEYiXvsWMNJD+xPwdLly4NYjfddFOxL0eypDtuEZHEaOIWEUmMJm4RkcRo4hYRScwpFyfNrCuAPzXlHgQwBcD7ADY2pdzt7iuKdoUiRaBxLSmzU5W2mtmPAax399fM7HEAOwB0cvdfZPMEI0aM8Dlz5rT+mTSX7VqI7ehgHccvuOACmrtv374gFrsGlhvres6aEMR2WQwbNiyIbd26NYjFSuZZl/a3336b5rIGC7Gu9qxLe2xHx+LFi4PY0KFDaS57vljTBdb4IXZ8AfsszGypu2foAyLaOq4BIJPJeF1dXS5PK5K1TCaDuro6OlFl0+X9sS/89hwA2wBMNrMrAWwBcIe78/1jImVK41pSlvV33GY2FkA3AK8BmODu4wHsA/ANkjvVzOrMrC52BypSDnIZ1035n43tWFs9kWLLauI2s+4Afg/gTgDvunvzMW9rAAT/Vnb3me6ecfcMOyVOpBzkOq6BlmM7dpKjSLGdcuI2s2oAzwO41923AJhjZrVm1g7AZADvFPkaRQpO41pSlk3J+/cBjAEwzcymAZgHYA4AA/A3d//nyR7c2NiIw4cPt4hVV1fT3IsuuiiIxcqk2dndsYU9tpAZ+5dAz549g9i8efNoLnsdsZJ1Vt6+YkW4aSH23qxcuTKIsQVPAFi7dm0Q+8pXvkJzWQf72MLt+PHjaZxhHezZuegAcPHFFwex2Pnjsa7keWjTuBY5nbJZnHwcwOOtwr8pzuWIlIbGtaRMBTgiIonRxC0ikhhN3CIiidHELSKSmKI3UjjzzDODcu1YmT3rIh7rxs52f7CGCQA/qD9m27ZtQSy2y4LteIntnPjHP/4RxDp37hzEYiXk7733XlaPB3gjhFjJO9sFE9u5wZousPcL4N3YYw0a2DEDsc+MNYkQ+bLRHbeISGI0cYuIJEYTt4hIYjRxi4gk5pTncbf5Ccw+woljMgGgJ4Dw0O30VerrAtJ4bQPcveQnPmlsJy2F1xUd10WfuFs8mVldrgfep6BSXxdQ2a+tkCr1fdLrKk/6qkREJDGauEVEElPqiXtmiZ+vVCr1dQGV/doKqVLfJ72uMlTS77hFRKTt9FWJiEhiNHGLiCSmZBO3mc02szfNbHqpnrOYzKyXmS1o+nV7M3up6fXdebqvLV9m1tXM/m5mr5nZC2ZWXWmfWzFU2nuksV3+SjJxm9mNANq5+zgAfcyMH4GXCDPrBuBpAJ2aQncDqGt6fTeYWXiMXhpuBTDD3ScB2AngO6igz60YNLaTUVFju1R33BNxoqM2AMwFkH3X2fJ0HMAUAM1npU7E56/vTQBJbux398fc/bWm354D4DZU1udWDBNRWe+RxnYCSjVxdwKwvenX9QB6leh5i8Ld6939iy3lK+r1mdlYAN0AbEMFva4iqajPXmM7DaWauA8CaD4Zv3MJn7dUKub1mVl3AL8HcCcq6HUVUaW/RxXz+ippbJfqYpfi83+K1ALYXKLnLZWKeH1mVo0T/3y81923oEJeV5FV+ntUEa+v0sZ20VuXNXkRwAIz6wPg6wCuKNHzlsrTAF42s6sAjACw+DRfT76+D2AMgGlmNg3AHwDcXsGfWyG8CI3tFFTU2C5Z5WTTavUkAPPdfWdJnrSEmgbAeACvtvqOMGmV/rkVQqW/Rxrb5Ucl7yIiiUnqC3kREdHELSKSHE3cIiKJ0cQtIpIYTdwiIon5f1Y5q/WpWJPCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "some_index = 5500\n",
    "plt.subplot(121);plt.imshow(X_test_mod[some_index].reshape(28, 28), cmap = matplotlib.cm.binary)\n",
    "plt.subplot(122);plt.imshow(y_test_mod[some_index].reshape(28, 28), cmap = matplotlib.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD2CAYAAAD720p7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAMZElEQVR4nO3dX4hc9RnG8edpUlHzx6Q4hCYXzYUJpVADdsS4JDTVhqhE1PYiQuuNhVwEvG5LVLRYlF7UQiEpKxpCSCretEg0BDWK0VhxNrRWwdJaTBttYIWYTUDU6tuLPZLtZvfM7Jkzf/Lu9wNLzpzfnDmvx/PwO/9mfo4IAcjpK4MuAEDvEHAgMQIOJEbAgcQIOJDYwl6v4Morr4zVq1f3ejXAvDY2NvZhRDSmz+95wFevXq1Wq9Xr1QDzmu0TM82vfIhu+3Hbx2zfW70sAL1UKeC2fyBpQUSMSFppe029ZQGoQ9UefJOkp4rpI5I2TG20vd12y3ZrfHy8i/IAdKNqwBdJer+YnpC0YmpjRIxGRDMimo3GBef9APqkasDPSbqsmF7cxecA6KGqwRzT+cPydZLeq6UaALWqepvsj5KO2l4p6WZJ62urCEBtKvXgETGhyQttf5L0vYg4U2dRAOpR+UGXiDit81fSAQwhLo4BiRFwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHEKg8+CAyjsbGxWduWLFlSuuzatWvrLmfg5tyD215o+1+2Xyr+vt2LwgB0r0oPfrWk30fET+suBkC9qpyDr5d0h+1XbO+3zWE+MKSqBPwNSd+NiA2SPpJ0y/Q32N5uu2W7NT4+3mWJAKqqEvA3I+I/xfQ7ktZMf0NEjEZEMyKajUajqwIBVFcl4Ptsr7O9QNIdkv5Sc00AalLl/PkXkg5IsqSnI+L5eksCUJc5Bzwi3tLklXSg77744ovS9muvvXbWtj179pQuy31wABcVAg4kRsCBxAg4kBgBBxIj4EBiPEeOi8qCBQtK21999dVZ20ZGRuouZ+jRgwOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYtwHR1+1+7pnu/vct912W2n7fLzXXYYeHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcS4z74gHzyySel7Zdeemlpe9kwuddcc02lmvrh+PHjXS3/2GOP1VTJ/EAPDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJcR98QNrd57Zd2j7M97rPnj07a1vZ8L6S9OSTT5a2NxqNSjXNV/TgQGIdBdz2CttHi+mv2j5o+5jtu3tbHoButA247eWS9kpaVMy6R1IrIkYkbbW9pIf1AehCJz3455K2SZooXm+S9FQxfUxSc/oCtrfbbtlujY+P11EngAraBjwiJiLizJRZiyS9X0xPSFoxwzKjEdGMiCYXRYDBqXKR7Zyky4rpxRU/A0AfVAnnmKQNxfQ6Se/VVg2AWlW5D75X0rO2N0r6lqTX6y0ph3b3sbds2VLavn///jrL6aulS5dWXnbXrl2l7du2bav82fNRxz14RGwq/j0habOkVyV9PyI+701pALpV6Um2iPhA56+kAxhSXCADEiPgQGIEHEiMgAOJ8XXRitrdBrv11ltL2w8cOFDavnjx4jnX1C+jo6Ol7WXb5oEHHihd9v77769SEmZBDw4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiTkierqCZrMZrVarp+sYhHb3wds5ffp0afuyZcu6+vxuvPvuu6XtV111VeXP/vjjj0vb2/2cNGZmeywiLvj5NHpwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiM74MPyPLly0vb9+3bV9p+ww03zNp25MiR0mXffvvt0vZHHnmktL0bTzzxRGl7u+cLduzYUdpe9rPL7ZY9efJkafuqVatK24cRPTiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJMb3wSvaunVrafszzzzTp0rml3b3ycv250OHDpUue9NNN1WqaRh09X1w2ytsHy2mV9k+aful4q9Rd7EA6tH2STbbyyXtlbSomHWdpF9GxO5eFgage5304J9L2iZponi9XtIO26/ZfrRnlQHoWtuAR8RERJyZMuuQpJGIuF7SWttXT1/G9nbbLdut8fHxGssFMBdVrqIfi4izxfQ7ktZMf0NEjEZEMyKajQan6MCgVAn4Ydtft325pC2S3qq5JgA1qfJ10QclvSjpU0m/i4i/1VsSgLp0HPCI2FT8+6Kkb/aqoIvFwYMHS9uPHz9e2n7jjTeWtr/wwgtzrulLzeYFt0P/T7tnH9rda7799ttL28u+qz4yMlK67NKlS0vbu/lN9vmIJ9mAxAg4kBgBBxIj4EBiBBxIjIADifF10Xmo26GPe73PYO4YPhiYhwg4kBgBBxIj4EBiBBxIjIADiRFwIDGGD07ooYce6mr506dP11QJBo0eHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcS4z74RerRR2cfFu6+++4rXfbll18ubV+2bFmVkjCE6MGBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDHugw+pDz74oLT94YcfrvzZGzdurLwsLi5te3DbV9g+ZPs523+wfYntx20fs31vP4oEUE0nh+g/kvTriNgs6ZSkOyUtiIgRSSttr+llgQCqa3uIHhG7prxsSPqxpN8Ur49I2iDp77VXBqBrHV9ks329pOWS/i3p/WL2hKQVM7x3u+2W7db4+HgthQKYu44Cbvtrkn4r6W5J5yRdVjQtnukzImI0IpoR0Ww0GnXVCmCOOrnIdomkpyT9PCJOSBrT5GG5JK2T9F7PqgPQlU5uk/1E0nck7bS9U9IeSXfZXinpZknre1jfvLV3797S9rJTn88++6zucnCR6uQi225Ju6fOs/20pM2SfhURZ3pUG4AuVXrQJSJOa/KwHcAQ41FVIDECDiRGwIHECDiQGAEHEuProgNy6tSp0vadO3dW/uyFC/nfikn04EBiBBxIjIADiRFwIDECDiRGwIHECDiQGDdMB+Tw4cOl7RFR2s5PYaET9OBAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kJjb3W/tVrPZjFar1dN1APOd7bGIaE6fTw8OJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCCxtj/4YPsKSU8W7z0naZukf0j6Z/GWeyLirz2rEEBlnfTgP5L064jYLOmUpJ9J+n1EbCr+CDcwpNoGPCJ2RcRzxcuGpP9KusP2K7b3277gKMD2dtst2y1+WggYnI7PwW1fL2m5pOckfTciNkj6SNIt098bEaMR0YyIZqPRqKtWAHPU0Y8u2v6apN9K+qGkUxHxSdH0jqQ1PaoNQJfa9uC2L5H0lKSfR8QJSftsr7O9QNIdkv7S4xoBVNTJIfpPJH1H0k7bL0l6W9I+SX+W9FpEPN+z6gB0pe0hekTslrR72uwHe1MOgDrxoAuQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCCxng8fbHtc0okps66U9GFPV1odtVVDbXNXd13fiIgLfh+t5wG/YIV2a6ZxjIcBtVVDbXPXr7o4RAcSI+BAYoMI+OgA1tkpaquG2uauL3X1/RwcQP9wiA4kRsCBxPoacNuP2z5m+95+rrcd2wtt/8v2S8XftwddkyTZXmH7aDH9VdsHi+1395DVtsr2ySnbbyDjVdm+wvYh28/Z/oPtS4Zln5ultp7vc30LuO0fSFoQESOSVtoepiGPrtaQjZhqe7mkvZIWFbPukdQqtt9W20uGqLbrJP1yyvYb1IiT00fCvVPDs88NZJTefvbgmzQ5BJIkHZG0oY/rbme92oyYOgCfa3Is9oni9Sad337HJA3y4Y3pta2XtMP2a7YfHVRRM4yE+2MNyT5XZZTeOvQz4IskvV9MT0ha0cd1t/OG2oyY2m8RMRERZ6bMGprtN0NthySNRMT1ktbavnpApUn6v5Fw/60h2WZfmssovXXoZ8DPSbqsmF7c53W382ZE/KeYHtYRU4d5+x2LiLPF9EC335SRcO/WkG2zabX1ZZ/r53/wmM4fIq2T9F4f193OxTBi6jBvv8O2v277cklbJL01iCJmGAl3aLbZoEbp7ee55h8lHbW9UtLNmjxvGxa/kHRAkiU9PaQjpu6V9KztjZK+Jen1Adcz1YOSXpT0qaTfRcTfBlTH1JFwd0raI+muIdnnptf2oiZH6e3pPtfXJ9mKq6+bJb0cEaf6tuIkih11g6TD086BMYv5vs/xqCqQ2DBdqAFQMwIOJEbAgcQIOJAYAQcS+x9x9UqdcVtVHQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "knn_clf.fit(X_train_mod, y_train_mod)\n",
    "clean_digit = knn_clf.predict([X_test_mod[some_index]])\n",
    "\n",
    "plt.imshow(clean_digit.reshape(28, 28), cmap = matplotlib.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
